From cd756d2044ef20a0c082872dc6caf7a7fc33c4bb Mon Sep 17 00:00:00 2001 From: csthomas1 Date: Tue, 21 Feb 2023 18:55:32 +0000 Subject: [PATCH 001/404] issue#19583 -- Make KubernetesRuntime translate tenant, namespace, and function name characters that are not allowed within Kubernetes labels as part of function removal --- .../functions/runtime/kubernetes/KubernetesRuntime.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index e206862b68a67..5d462dda6e01b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -684,10 +684,11 @@ public void deleteStatefulSet() throws InterruptedException { .numRetries(KubernetesRuntimeFactory.numRetries * 2) .sleepBetweenInvocationsMs(KubernetesRuntimeFactory.sleepBetweenRetriesMs * 2) .supplier(() -> { + Map validLabels = getLabels(instanceConfig.getFunctionDetails()); String labels = String.format("tenant=%s,namespace=%s,name=%s", - instanceConfig.getFunctionDetails().getTenant(), - instanceConfig.getFunctionDetails().getNamespace(), - instanceConfig.getFunctionDetails().getName()); + validLabels.get("tenant"), + validLabels.get("namespace"), + validLabels.get("name")); V1PodList response; try { From 660525e57ed35b74cb9204521d1fba02cc08c542 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 22 Feb 2023 07:58:51 +0800 Subject: [PATCH 002/404] [improve][fn] Support schema in python instance (#18432) --- .../worker/PulsarFunctionLocalRunTest.java | 2 +- .../common/functions/FunctionConfig.java | 2 + .../apache/pulsar/admin/cli/CmdFunctions.java | 12 +++ .../src/main/python/python_instance.py | 87 +++++++++++++-- .../api/examples/pojo/AvroTestObject.java | 2 +- .../avro_schema_test_function.py | 40 +++++++ .../functions/utils/FunctionConfigUtils.java | 4 + .../functions/PulsarFunctionsTest.java | 101 ++++++++++++------ .../functions/PulsarFunctionsTestBase.java | 2 + .../java/PulsarFunctionsJavaTest.java | 2 +- .../python/PulsarFunctionsPythonTest.java | 5 + .../functions/utils/CommandGenerator.java | 20 ++++ 12 files changed, 234 insertions(+), 45 deletions(-) create mode 100644 pulsar-functions/python-examples/avro_schema_test_function.py diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index 1d98ec1e2515c..c832cba163d63 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -670,7 +670,7 @@ private void testAvroFunctionLocalRun(String jarFilePathUrl) throws Exception { }, 50, 150); int totalMsgs = 5; - Method setBaseValueMethod = avroTestObjectClass.getMethod("setBaseValue", new Class[]{int.class}); + Method setBaseValueMethod = avroTestObjectClass.getMethod("setBaseValue", new Class[]{Integer.class}); for (int i = 0; i < totalMsgs; i++) { Object avroTestObject = avroTestObjectClass.getDeclaredConstructor().newInstance(); setBaseValueMethod.invoke(avroTestObject, i); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java index 2dddb67c9503c..0b26e7e93b5f0 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java @@ -72,6 +72,7 @@ public enum Runtime { * A generalized way of specifying inputs. */ private Map inputSpecs; + private String inputTypeClassName; private String output; @@ -83,6 +84,7 @@ public enum Runtime { * implementation. */ private String outputSchemaType; + private String outputTypeClassName; private String outputSerdeClassName; private String logTopic; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index a9c2f91d2b2c2..bc2585bc67bc9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -269,6 +269,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--input-specs", description = "The map of inputs to custom configuration (as a JSON string) #Java, Python, Go") protected String inputSpecs; + @Parameter(names = "--input-type-class-name", + description = "The class name of input type class #Java, Python, Go") + protected String inputTypeClassName; // for backwards compatibility purposes @Parameter(names = "--outputSerdeClassName", description = "The SerDe class to be used for messages output by the function", hidden = true) @@ -276,6 +279,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--output-serde-classname", description = "The SerDe class to be used for messages output by the function #Java, Python") protected String outputSerdeClassName; + @Parameter(names = "--output-type-class-name", + description = "The class name of output type class #Java, Python, Go") + protected String outputTypeClassName; // for backwards compatibility purposes @Parameter(names = "--functionConfigFile", description = "The path to a YAML config file that specifies " + "the configuration of a Pulsar Function", hidden = true) @@ -485,12 +491,18 @@ void processArguments() throws Exception { Type type = new TypeToken>() {}.getType(); functionConfig.setInputSpecs(new Gson().fromJson(inputSpecs, type)); } + if (null != inputTypeClassName) { + functionConfig.setInputTypeClassName(inputTypeClassName); + } if (null != topicsPattern) { functionConfig.setTopicsPattern(topicsPattern); } if (null != output) { functionConfig.setOutput(output); } + if (null != outputTypeClassName) { + functionConfig.setOutputTypeClassName(outputTypeClassName); + } if (null != producerConfig) { Type type = new TypeToken() {}.getType(); functionConfig.setProducerConfig(new Gson().fromJson(producerConfig, type)); diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index cf53e75d9d0cf..f77ef38f76e4d 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -32,6 +32,7 @@ import threading import sys import re +import inspect import pulsar import contextimpl import Function_pb2 @@ -50,9 +51,10 @@ # Equivalent of the InstanceConfig in Java InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_details max_buffered_tuples') # This is the message that the consumers put on the queue for the function thread to process -InternalMessage = namedtuple('InternalMessage', 'message topic serde consumer') +InternalMessage = namedtuple('InternalMessage', 'message topic serde use_schema consumer') InternalQuitMessage = namedtuple('InternalQuitMessage', 'quit') DEFAULT_SERIALIZER = "serde.IdentitySerDe" +DEFAULT_SCHEMA = pulsar.schema.BytesSchema() PY3 = sys.version_info[0] >= 3 @@ -93,8 +95,10 @@ def __init__(self, self.pulsar_client = pulsar_client self.state_storage_serviceurl = state_storage_serviceurl self.input_serdes = {} + self.input_schema = {} self.consumers = {} self.output_serde = None + self.output_schema = DEFAULT_SCHEMA self.function_class = None self.function_purefunction = None self.producer = None @@ -166,7 +170,7 @@ def run(self): self.consumers[topic] = self.pulsar_client.subscribe( str(topic), subscription_name, consumer_type=mode, - message_listener=partial(self.message_listener, self.input_serdes[topic]), + message_listener=partial(self.message_listener, self.input_serdes[topic], DEFAULT_SCHEMA), unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None, initial_position=position, properties=properties @@ -178,11 +182,16 @@ def run(self): else: serde_kclass = util.import_class(os.path.dirname(self.user_code), consumer_conf.serdeClassName) self.input_serdes[topic] = serde_kclass() + + self.input_schema[topic] = self.get_schema(consumer_conf.schemaType, + self.instance_config.function_details.source.typeClassName, + consumer_conf.schemaProperties) Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) consumer_args = { "consumer_type": mode, - "message_listener": partial(self.message_listener, self.input_serdes[topic]), + "schema": self.input_schema[topic], + "message_listener": partial(self.message_listener, self.input_serdes[topic], self.input_schema[topic]), "unacked_messages_timeout_ms": int(self.timeout_ms) if self.timeout_ms else None, "initial_position": position, "properties": properties @@ -233,8 +242,10 @@ def actual_execution(self): if isinstance(msg, InternalQuitMessage): break Log.debug("Got a message from topic %s" % msg.topic) - # deserialize message - input_object = msg.serde.deserialize(msg.message.data()) + input_object = msg.message.value() + if not msg.use_schema: + # deserialize message + input_object = msg.serde.deserialize(msg.message.data()) # set current message in context self.contextimpl.set_current_message_context(msg.message, msg.topic) output_object = None @@ -296,12 +307,14 @@ def process_result(self, output, msg): if self.producer is None: self.setup_producer() - # serialize function output - output_bytes = self.output_serde.serialize(output) + # only serialize function output when output schema is not set + output_object = output + if self.output_schema == DEFAULT_SCHEMA: + output_object = self.output_serde.serialize(output) - if output_bytes is not None: + if output_object is not None: props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())} - self.producer.send_async(output_bytes, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props) + self.producer.send_async(output_object, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props) elif self.auto_ack and self.atleast_once: msg.consumer.acknowledge(msg.message) @@ -327,8 +340,13 @@ def setup_producer(self): if batch_builder == "KEY_BASED": batch_type = pulsar.BatchingType.KeyBased + self.output_schema = self.get_schema(self.instance_config.function_details.sink.schemaType, + self.instance_config.function_details.sink.typeClassName, + self.instance_config.function_details.sink.schemaProperties) + self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), + schema=self.output_schema, block_if_queue_full=True, batching_enabled=True, batching_type=batch_type, @@ -351,10 +369,13 @@ def setup_state(self): table_name = str(self.instance_config.function_details.name) return state_context.create_state_context(self.state_storage_serviceurl, table_ns, table_name) - def message_listener(self, serde, consumer, message): + def message_listener(self, serde, schema, consumer, message): # increment number of received records from source self.stats.incr_total_received() - item = InternalMessage(message, message.topic_name(), serde, consumer) + use_schema = False + if schema != DEFAULT_SCHEMA: + use_schema = True + item = InternalMessage(message, message.topic_name(), serde, use_schema, consumer) self.queue.put(item, True) if self.atmost_once and self.auto_ack: consumer.acknowledge(message) @@ -462,3 +483,47 @@ def close(self): if self.pulsar_client: self.pulsar_client.close() + + + # TODO: support other schemas: PROTOBUF, PROTOBUF_NATIVE, and KeyValue + def get_schema(self, schema_type, type_class_name, schema_properties): + schema = DEFAULT_SCHEMA + if schema_type == "" or schema_type is None: + schema = DEFAULT_SCHEMA + elif schema_type.lower() == "string": + schema = pulsar.schema.StringSchema() + elif schema_type.lower() == "json": + record_kclass = self.get_record_class(type_class_name) + schema = pulsar.schema.JsonSchema(record_kclass) + elif schema_type.lower() == "avro": + record_kclass = self.get_record_class(type_class_name) + schema = pulsar.schema.AvroSchema(record_kclass) + else: # load custom schema + record_kclass = self.get_record_class(type_class_name) + schema_kclass = util.import_class(os.path.dirname(self.user_code), schema_type) + args_count = 0 + try: + args_count = len(inspect.signature(schema_kclass.__init__).parameters) + except: # for compatibility with python 2 + args_count = len(inspect.getargspec(schema_kclass.__init__).args) + if args_count == 1: # doesn't take any arguments + schema = schema_kclass() + elif args_count == 2: # take one argument, it can be either schema properties or record class + try: + schema = schema_kclass(record_kclass) + except TypeError: + schema = schema_kclass(schema_properties) + elif args_count >= 3: # take two or more arguments + schema = schema_kclass(record_kclass, schema_properties) + else: + raise Exception("Invalid schema class %s" % schema_type) + return schema + + def get_record_class(self, class_name): + record_kclass = None + if class_name != None and len(class_name) > 0: + try: + record_kclass = util.import_class(os.path.dirname(self.user_code), class_name) + except: + pass + return record_kclass \ No newline at end of file diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java index 4967dbac9dae4..e7abb8e7d6ed0 100644 --- a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java @@ -27,7 +27,7 @@ @Data public class AvroTestObject { - private int baseValue; + private Integer baseValue; private String objectValue; } diff --git a/pulsar-functions/python-examples/avro_schema_test_function.py b/pulsar-functions/python-examples/avro_schema_test_function.py new file mode 100644 index 0000000000000..46704439a93ca --- /dev/null +++ b/pulsar-functions/python-examples/avro_schema_test_function.py @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +from pulsar import Function +from pulsar.schema import * + + +class AvroTestObject(Record): + def __init__(self, baseValue, objectValue): + self.baseValue = baseValue + self.objectValue = objectValue + + baseValue = Integer() + objectValue = String() + + +class AvroSchemaTestFunction(Function): + def __init__(self): + pass + + def process(self, input, context): + input.__setattr__("baseValue", input.baseValue + 10) + return input diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index f6a5da1177fc4..647210de33acf 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -200,6 +200,8 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu if (extractedDetails.getTypeArg0() != null) { sourceSpecBuilder.setTypeClassName(extractedDetails.getTypeArg0()); + } else if (StringUtils.isNotEmpty(functionConfig.getInputTypeClassName())) { + sourceSpecBuilder.setTypeClassName(functionConfig.getInputTypeClassName()); } if (functionConfig.getTimeoutMs() != null) { sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs()); @@ -242,6 +244,8 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu } if (extractedDetails.getTypeArg1() != null) { sinkSpecBuilder.setTypeClassName(extractedDetails.getTypeArg1()); + } else if (StringUtils.isNotEmpty(functionConfig.getOutputTypeClassName())) { + sinkSpecBuilder.setTypeClassName(functionConfig.getOutputTypeClassName()); } if (functionConfig.getProducerConfig() != null) { ProducerConfig producerConf = functionConfig.getProducerConfig(); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 7a88d35a65ac3..eaf66974b982a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.time.Duration; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; @@ -60,6 +61,7 @@ import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; +import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.policies.data.FunctionStatsImpl; import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.FunctionStatusUtil; @@ -174,8 +176,8 @@ protected void testFunctionLocalRun(Runtime runtime) throws Exception { assertEquals(admin.topics().getStats(inputTopicName).getSubscriptions().size(), 1); // publish and consume result - if (Runtime.JAVA == runtime) { - // java supports schema + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { + // java and python supports schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) .build(); @@ -208,7 +210,7 @@ protected void testFunctionLocalRun(Runtime runtime) throws Exception { assertEquals(expectedMessages.size(), 0); } else { - // python doesn't support schema + // golang doesn't support schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) @@ -355,7 +357,7 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { Schema schema; - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { schema = Schema.STRING; } else { schema = Schema.BYTES; @@ -388,8 +390,8 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { getFunctionStatsEmpty(functionName); // publish and consume result - if (Runtime.JAVA == runtime) { - // java supports schema + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { + // java and python supports schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) .build(); @@ -420,7 +422,7 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { assertEquals(expectedMessages.size(), 0); } else { - // python doesn't support schema + // golang doesn't support schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) @@ -530,7 +532,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { } Schema schema; - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { schema = Schema.STRING; } else { schema = Schema.BYTES; @@ -559,9 +561,14 @@ protected void testPublishFunction(Runtime runtime) throws Exception { PUBLISH_JAVA_CLASS, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null); + null, null, null, null, null); break; case PYTHON: + ConsumerConfig consumerConfig = new ConsumerConfig(); + consumerConfig.setSchemaType("string"); + Map inputSpecs = new HashMap<>() {{ + put(inputTopicName, objectMapper.writeValueAsString(consumerConfig)); + }}; submitFunction( runtime, inputTopicName, @@ -571,7 +578,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { PUBLISH_PYTHON_CLASS, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null); + objectMapper.writeValueAsString(inputSpecs), "string", null, null, null); break; case GO: submitFunction( @@ -583,7 +590,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { null, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null); + null, null, null, null, null); } // get function info @@ -594,11 +601,11 @@ protected void testPublishFunction(Runtime runtime) throws Exception { // publish and consume result - if (Runtime.JAVA == runtime) { - // java supports schema + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { + // java and python supports schema publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); } else { - // python doesn't support schema. Does Go? Maybe we need a switch instead for the Go case. + // Does Go support schema? Maybe we need a switch instead for the Go case. @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) @@ -663,7 +670,7 @@ protected void testExclamationFunction(Runtime runtime, Schema schema; - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { schema = Schema.STRING; } else { schema = Schema.BYTES; @@ -707,11 +714,11 @@ protected void testExclamationFunction(Runtime runtime, getFunctionStatsEmpty(functionName); // publish and consume result - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { // java supports schema publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); } else { - // python doesn't support schema + // golang doesn't support schema publishAndConsumeMessagesBytes(inputTopicName, outputTopicName, numMessages); } @@ -793,7 +800,7 @@ private void submitFunction(Runtime runtime, String functionClass, Schema inputTopicSchema) throws Exception { submitFunction(runtime, inputTopicName, outputTopicName, functionName, functionFile, functionClass, - inputTopicSchema, null, null, null, null); + inputTopicSchema, null, null, null, null, null, null); } private void submitFunction(Runtime runtime, @@ -806,7 +813,9 @@ private void submitFunction(Runtime runtime, Map userConfigs, String customSchemaInputs, String outputSchemaType, - SubscriptionInitialPosition subscriptionInitialPosition) throws Exception { + SubscriptionInitialPosition subscriptionInitialPosition, + String inputTypeClassName, + String outputTypeClassName) throws Exception { if (StringUtils.isNotEmpty(inputTopicName)) { ensureSubscriptionCreated( @@ -836,6 +845,12 @@ private void submitFunction(Runtime runtime, if (subscriptionInitialPosition != null) { generator.setSubscriptionInitialPosition(subscriptionInitialPosition); } + if (inputTypeClassName != null) { + generator.setInputTypeClassName(inputTypeClassName); + } + if (outputTypeClassName != null) { + generator.setOutputTypeClassName(outputTypeClassName); + } String command = ""; switch (runtime){ @@ -857,6 +872,8 @@ private void submitFunction(Runtime runtime, }; ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( commands); + log.info("---------- stdout is: {}", result.getStdout()); + log.info("---------- stderr is: {}", result.getStderr()); assertTrue(result.getStdout().contains("Created successfully")); } @@ -1326,7 +1343,12 @@ private void publishAndConsumeAvroMessages(String inputTopic, } } - protected void testAvroSchemaFunction() throws Exception { + protected void testAvroSchemaFunction(Runtime runtime) throws Exception { + if (functionRuntimeType == FunctionRuntimeType.THREAD && runtime == Runtime.PYTHON) { + // python can only run on process mode + return; + } + log.info("testAvroSchemaFunction start ..."); final String inputTopic = "test-avroschema-input-" + randomName(8); final String outputTopic = "test-avroschema-output-" + randomName(8); @@ -1375,14 +1397,31 @@ protected void testAvroSchemaFunction() throws Exception { } }); - submitFunction( - Runtime.JAVA, - inputTopic, - outputTopic, - functionName, - null, - AvroSchemaTestFunction.class.getName(), - Schema.AVRO(AvroTestObject.class)); + if (runtime == Runtime.JAVA) { + submitFunction( + Runtime.JAVA, + inputTopic, + outputTopic, + functionName, + null, + AvroSchemaTestFunction.class.getName(), + Schema.AVRO(AvroTestObject.class)); + } else if (runtime == Runtime.PYTHON) { + ConsumerConfig consumerConfig = new ConsumerConfig(); + consumerConfig.setSchemaType("avro"); + Map inputSpecs = new HashMap<>() {{ + put(inputTopic, objectMapper.writeValueAsString(consumerConfig)); + }}; + submitFunction( + Runtime.PYTHON, + inputTopic, + outputTopic, + functionName, + AVRO_SCHEMA_FUNCTION_PYTHON_FILE, + AVRO_SCHEMA_PYTHON_CLASS, + Schema.AVRO(AvroTestObject.class), + null, objectMapper.writeValueAsString(inputSpecs), "avro", null, "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); + } log.info("pulsar submitFunction"); getFunctionInfoSuccess(functionName); @@ -1457,7 +1496,7 @@ protected void testInitFunction(Runtime runtime) throws Exception { // submit the exclamation function submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, InitializableFunction.class.getName(), schema, - Collections.singletonMap("publish-topic", outputTopicName), null, null, null); + Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null); // publish and consume result publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); @@ -1645,7 +1684,7 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField null, null, SchemaType.NONE.name(), - SubscriptionInitialPosition.Earliest); + SubscriptionInitialPosition.Earliest, null, null); try { if (keyValue) { @Cleanup @@ -1830,7 +1869,7 @@ protected void testMergeFunction() throws Exception { null, inputSpecNode.toString(), SchemaType.AUTO_PUBLISH.name().toUpperCase(), - SubscriptionInitialPosition.Earliest); + SubscriptionInitialPosition.Earliest, null, null); getFunctionInfoSuccess(functionName); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 71f7a861cd8e2..033e590d6dd4e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -68,11 +68,13 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { public static final String PUBLISH_PYTHON_CLASS = "typed_message_builder_publish.TypedMessageBuilderPublish"; public static final String EXCEPTION_PYTHON_CLASS = "exception_function"; + public static final String AVRO_SCHEMA_PYTHON_CLASS = "avro_schema_test_function.AvroSchemaTestFunction"; public static final String EXCLAMATION_PYTHON_FILE = "exclamation_function.py"; public static final String EXCLAMATION_WITH_DEPS_PYTHON_FILE = "exclamation_with_extra_deps.py"; public static final String EXCLAMATION_PYTHON_ZIP_FILE = "exclamation.zip"; public static final String PUBLISH_FUNCTION_PYTHON_FILE = "typed_message_builder_publish.py"; public static final String EXCEPTION_FUNCTION_PYTHON_FILE = "exception_function.py"; + public static final String AVRO_SCHEMA_FUNCTION_PYTHON_FILE = "avro_schema_test_function.py"; public static final String EXCLAMATION_GO_FILE = "exclamationFunc"; public static final String PUBLISH_FUNCTION_GO_FILE = "exclamationFunc"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java index 6828630940e2e..e6f3c67ebcdb7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -210,7 +210,7 @@ public void testAutoSchemaFunctionTest() throws Exception { @Test(groups = {"java_function", "function"}) public void testAvroSchemaFunctionTest() throws Exception { - testAvroSchemaFunction(); + testAvroSchemaFunction(Runtime.JAVA); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java index 909da3d28583e..1c75d70498609 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java @@ -64,4 +64,9 @@ public void testPythonExclamationTopicPatternFunction() throws Exception { testExclamationFunction(Runtime.PYTHON, true, false, false); } + @Test(groups = {"python_function", "function"}) + public void testAvroSchemaFunctionTest() throws Exception { + testAvroSchemaFunction(Runtime.PYTHON); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java index ab411e24dbe5f..90fac7a055268 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java @@ -60,6 +60,8 @@ public enum Runtime { private Integer slidingIntervalCount; private Long slidingIntervalDurationMs; private String customSchemaInputs; + private String inputTypeClassName; + private String outputTypeClassName; private String schemaType; private SubscriptionInitialPosition subscriptionInitialPosition; @@ -111,6 +113,12 @@ public String generateLocalRunCommand(String codeFile) { if (customSchemaInputs != null) { commandBuilder.append(" --custom-schema-inputs \'" + customSchemaInputs + "\'"); } + if (inputTypeClassName != null) { + commandBuilder.append(" --input-type-class-name " + inputTypeClassName); + } + if (outputTypeClassName != null) { + commandBuilder.append(" --output-type-class-name " + outputTypeClassName); + } if (schemaType != null) { commandBuilder.append(" --schema-type " + schemaType); } @@ -207,6 +215,12 @@ public String generateCreateFunctionCommand(String codeFile) { if (customSchemaInputs != null) { commandBuilder.append(" --custom-schema-inputs \'" + customSchemaInputs + "\'"); } + if (inputTypeClassName != null) { + commandBuilder.append(" --input-type-class-name " + inputTypeClassName); + } + if (outputTypeClassName != null) { + commandBuilder.append(" --output-type-class-name " + outputTypeClassName); + } if (schemaType != null) { commandBuilder.append(" --schema-type " + schemaType); } @@ -305,6 +319,12 @@ public String generateUpdateFunctionCommand(String codeFile) { if (customSchemaInputs != null) { commandBuilder.append(" --custom-schema-inputs \'" + customSchemaInputs + "\'"); } + if (inputTypeClassName != null) { + commandBuilder.append(" --input-type-class-name " + inputTypeClassName); + } + if (outputTypeClassName != null) { + commandBuilder.append(" --output-type-class-name " + outputTypeClassName); + } if (schemaType != null) { commandBuilder.append(" --schema-type " + schemaType); } From b38556aa19a1b29accc3b2d64170d169e80ce135 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 22 Feb 2023 10:18:13 +0800 Subject: [PATCH 003/404] [fix][broker] Fix geo-replication admin (#19548) Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/BrokerService.java | 92 +++++++++++++----- .../service/ReplicatorAdminTlsTest.java | 69 +++++++++++++ .../ReplicatorAdminTlsWithKeyStoreTest.java | 74 ++++++++++++++ .../broker/service/ReplicatorTestBase.java | 96 +++++++++++++++---- .../broker/service/ReplicatorTlsTest.java | 16 +++- 5 files changed, 303 insertions(+), 44 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index db7a3f16f97f2..b9b63427b370e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1370,6 +1370,29 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, } } + private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean brokerClientTlsEnabledWithKeyStore, + boolean isTlsAllowInsecureConnection, + String brokerClientTlsTrustStoreType, String brokerClientTlsTrustStore, + String brokerClientTlsTrustStorePassword, String brokerClientTlsKeyStoreType, + String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, + String brokerClientTrustCertsFilePath, + String brokerClientKeyFilePath, String brokerClientCertificateFilePath) { + if (brokerClientTlsEnabledWithKeyStore) { + adminBuilder.useKeyStoreTls(true) + .tlsTrustStoreType(brokerClientTlsTrustStoreType) + .tlsTrustStorePath(brokerClientTlsTrustStore) + .tlsTrustStorePassword(brokerClientTlsTrustStorePassword) + .tlsKeyStoreType(brokerClientTlsKeyStoreType) + .tlsKeyStorePath(brokerClientTlsKeyStore) + .tlsKeyStorePassword(brokerClientTlsKeyStorePassword); + } else { + adminBuilder.tlsTrustCertsFilePath(brokerClientTrustCertsFilePath) + .tlsKeyFilePath(brokerClientKeyFilePath) + .tlsCertificateFilePath(brokerClientCertificateFilePath); + } + adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection); + } + public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { PulsarAdmin admin = clusterAdmins.get(cluster); if (admin != null) { @@ -1379,37 +1402,58 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c try { ClusterData data = clusterDataOp .orElseThrow(() -> new MetadataStoreException.NotFoundException(cluster)); + PulsarAdminBuilder builder = PulsarAdmin.builder(); ServiceConfiguration conf = pulsar.getConfig(); - - boolean isTlsUrl = conf.isBrokerClientTlsEnabled() && isNotBlank(data.getServiceUrlTls()); - String adminApiUrl = isTlsUrl ? data.getServiceUrlTls() : data.getServiceUrl(); - PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(adminApiUrl); - // Apply all arbitrary configuration. This must be called before setting any fields annotated as // @Secret on the ClientConfigurationData object because of the way they are serialized. // See https://github.com/apache/pulsar/issues/8509 for more information. builder.loadConf(PropertiesUtils.filterAndMapProperties(conf.getProperties(), "brokerClient_")); - builder.authentication( - conf.getBrokerClientAuthenticationPlugin(), - conf.getBrokerClientAuthenticationParameters()); - - if (isTlsUrl) { - builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()); - if (conf.isBrokerClientTlsEnabledWithKeyStore()) { - builder.useKeyStoreTls(true) - .tlsTrustStoreType(conf.getBrokerClientTlsTrustStoreType()) - .tlsTrustStorePath(conf.getBrokerClientTlsTrustStore()) - .tlsTrustStorePassword(conf.getBrokerClientTlsTrustStorePassword()) - .tlsKeyStoreType(conf.getBrokerClientTlsKeyStoreType()) - .tlsKeyStorePath(conf.getBrokerClientTlsKeyStore()) - .tlsKeyStorePassword(conf.getBrokerClientTlsKeyStorePassword()); - } else { - builder.tlsTrustCertsFilePath(conf.getBrokerClientTrustCertsFilePath()) - .tlsKeyFilePath(conf.getBrokerClientKeyFilePath()) - .tlsCertificateFilePath(conf.getBrokerClientCertificateFilePath()); - } + if (data.getAuthenticationPlugin() != null && data.getAuthenticationParameters() != null) { + builder.authentication(data.getAuthenticationPlugin(), data.getAuthenticationParameters()); + } else { + builder.authentication(pulsar.getConfiguration().getBrokerClientAuthenticationPlugin(), + pulsar.getConfiguration().getBrokerClientAuthenticationParameters()); + } + + boolean isTlsEnabled = data.isBrokerClientTlsEnabled() || conf.isBrokerClientTlsEnabled(); + if (isTlsEnabled && StringUtils.isEmpty(data.getServiceUrlTls())) { + throw new IllegalArgumentException("serviceUrlTls is empty, brokerClientTlsEnabled: " + + isTlsEnabled); + } else if (StringUtils.isEmpty(data.getServiceUrl())) { + throw new IllegalArgumentException("serviceUrl is empty, brokerClientTlsEnabled: " + isTlsEnabled); + } + String adminApiUrl = isTlsEnabled ? data.getServiceUrlTls() : data.getServiceUrl(); + builder.serviceHttpUrl(adminApiUrl); + if (data.isBrokerClientTlsEnabled()) { + configAdminTlsSettings(builder, + data.isBrokerClientTlsEnabledWithKeyStore(), + data.isTlsAllowInsecureConnection(), + data.getBrokerClientTlsTrustStoreType(), + data.getBrokerClientTlsTrustStore(), + data.getBrokerClientTlsTrustStorePassword(), + data.getBrokerClientTlsKeyStoreType(), + data.getBrokerClientTlsKeyStore(), + data.getBrokerClientTlsKeyStorePassword(), + data.getBrokerClientTrustCertsFilePath(), + data.getBrokerClientKeyFilePath(), + data.getBrokerClientCertificateFilePath() + ); + } else if (conf.isBrokerClientTlsEnabled()) { + configAdminTlsSettings(builder, + conf.isBrokerClientTlsEnabledWithKeyStore(), + conf.isTlsAllowInsecureConnection(), + conf.getBrokerClientTlsTrustStoreType(), + conf.getBrokerClientTlsTrustStore(), + conf.getBrokerClientTlsTrustStorePassword(), + conf.getBrokerClientTlsKeyStoreType(), + conf.getBrokerClientTlsKeyStore(), + conf.getBrokerClientTlsKeyStorePassword(), + conf.getBrokerClientTrustCertsFilePath(), + conf.getBrokerClientKeyFilePath(), + conf.getBrokerClientCertificateFilePath() + ); } // most of the admin request requires to make zk-call so, keep the max read-timeout based on diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java new file mode 100644 index 0000000000000..a5d14ca0487dc --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ReplicatorAdminTlsTest extends ReplicatorTestBase { + + @Override + @BeforeClass(timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testReplicationAdmin() throws Exception { + for (BrokerService ns : List.of(ns1, ns2, ns3)) { + // load the admin + ns.getClusterPulsarAdmin(cluster1, Optional.of(admin1.clusters().getCluster(cluster1))); + ns.getClusterPulsarAdmin(cluster2, Optional.of(admin1.clusters().getCluster(cluster2))); + ns.getClusterPulsarAdmin(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); + + // verify the admin + ConcurrentOpenHashMap clusterAdmins = ns.getClusterAdmins(); + assertFalse(clusterAdmins.isEmpty()); + clusterAdmins.forEach((cluster, admin) -> { + ClientConfigurationData clientConfigData = ((PulsarAdminImpl) admin).getClientConfigData(); + assertEquals(clientConfigData.getTlsTrustCertsFilePath(), caCertFilePath); + assertEquals(clientConfigData.getTlsKeyFilePath(), clientKeyFilePath); + assertEquals(clientConfigData.getTlsCertificateFilePath(), clientCertFilePath); + assertTrue(clientConfigData.isUseTls()); + }); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java new file mode 100644 index 0000000000000..3d3eb3faa727f --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ReplicatorAdminTlsWithKeyStoreTest extends ReplicatorTestBase { + + @Override + @BeforeClass(timeOut = 300000) + public void setup() throws Exception { + tlsWithKeyStore = true; + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testReplicationAdmin() throws Exception { + for (BrokerService ns : List.of(ns1, ns2, ns3)) { + // load the admin + ns.getClusterPulsarAdmin(cluster1, Optional.of(admin1.clusters().getCluster(cluster1))); + ns.getClusterPulsarAdmin(cluster2, Optional.of(admin1.clusters().getCluster(cluster2))); + ns.getClusterPulsarAdmin(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); + + // verify the admin + ConcurrentOpenHashMap clusterAdmins = ns.getClusterAdmins(); + assertFalse(clusterAdmins.isEmpty()); + clusterAdmins.forEach((cluster, admin) -> { + ClientConfigurationData clientConfigData = ((PulsarAdminImpl) admin).getClientConfigData(); + assertEquals(clientConfigData.getTlsKeyStorePath(), clientKeyStorePath); + assertEquals(clientConfigData.getTlsKeyStorePassword(), keyStorePassword); + assertEquals(clientConfigData.getTlsKeyStoreType(), keyStoreType); + assertEquals(clientConfigData.getTlsTrustStorePath(), clientTrustStorePath); + assertEquals(clientConfigData.getTlsTrustStorePassword(), keyStorePassword); + assertEquals(clientConfigData.getTlsTrustStoreType(), keyStoreType); + assertTrue(clientConfigData.isUseKeyStoreTls()); + assertTrue(clientConfigData.isUseTls()); + }); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java index a7752b4a63fe2..b83e8ac9d2dbf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import com.google.common.io.Resources; import com.google.common.collect.Sets; import io.netty.util.concurrent.DefaultThreadFactory; @@ -87,8 +88,29 @@ public abstract class ReplicatorTestBase extends TestRetrySupport { static final int TIME_TO_CHECK_BACKLOG_QUOTA = 5; - protected static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; - protected static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + // PEM + protected final String brokerCertFilePath = Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + protected final String brokerFilePath = Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + protected final String clientCertFilePath = Resources.getResource("certificate-authority/client-keys/admin.cert.pem").getPath(); + protected final String clientKeyFilePath = Resources.getResource("certificate-authority/client-keys/admin.key-pk8.pem").getPath(); + protected final String caCertFilePath = Resources.getResource("certificate-authority/certs/ca.cert.pem").getPath(); + + // KEYSTORE + protected boolean tlsWithKeyStore = false; + protected final static String brokerKeyStorePath = + Resources.getResource("certificate-authority/jks/broker.keystore.jks").getPath(); + protected final static String brokerTrustStorePath = + Resources.getResource("certificate-authority/jks/broker.truststore.jks").getPath(); + protected final static String clientKeyStorePath = + Resources.getResource("certificate-authority/jks/client.keystore.jks").getPath(); + protected final static String clientTrustStorePath = + Resources.getResource("certificate-authority/jks/client.truststore.jks").getPath(); + protected final static String keyStoreType = "JKS"; + protected final static String keyStorePassword = "111111"; + + protected final String cluster1 = "r1"; + protected final String cluster2 = "r2"; + protected final String cluster3 = "r3"; // Default frequency public int getBrokerServicePurgeInactiveFrequency() { @@ -157,23 +179,56 @@ protected void setup() throws Exception { admin3 = PulsarAdmin.builder().serviceHttpUrl(url3.toString()).build(); // Provision the global namespace - admin1.clusters().createCluster("r1", ClusterData.builder() + admin1.clusters().createCluster(cluster1, ClusterData.builder() .serviceUrl(url1.toString()) .serviceUrlTls(urlTls1.toString()) .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) .build()); - admin1.clusters().createCluster("r2", ClusterData.builder() + admin1.clusters().createCluster(cluster2, ClusterData.builder() .serviceUrl(url2.toString()) .serviceUrlTls(urlTls2.toString()) .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) .build()); - admin1.clusters().createCluster("r3", ClusterData.builder() + admin1.clusters().createCluster(cluster3, ClusterData.builder() .serviceUrl(url3.toString()) .serviceUrlTls(urlTls3.toString()) .brokerServiceUrl(pulsar3.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) .build()); admin1.tenants().createTenant("pulsar", @@ -181,12 +236,12 @@ protected void setup() throws Exception { admin1.namespaces().createNamespace("pulsar/ns", Sets.newHashSet("r1", "r2", "r3")); admin1.namespaces().createNamespace("pulsar/ns1", Sets.newHashSet("r1", "r2")); - assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString()); - assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString()); - assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString()); - assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); - assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); - assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster(cluster1).getServiceUrl(), url1.toString()); + assertEquals(admin2.clusters().getCluster(cluster2).getServiceUrl(), url2.toString()); + assertEquals(admin2.clusters().getCluster(cluster3).getServiceUrl(), url3.toString()); + assertEquals(admin2.clusters().getCluster(cluster1).getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster(cluster2).getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster(cluster3).getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl()); // Also create V1 namespace for compatibility check admin1.clusters().createCluster("global", ClusterData.builder() @@ -194,7 +249,7 @@ protected void setup() throws Exception { .serviceUrlTls("https://global:8443") .build()); admin1.namespaces().createNamespace("pulsar/global/ns"); - admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Sets.newHashSet("r1", "r2", "r3")); + admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Sets.newHashSet(cluster1, cluster2, cluster3)); Thread.sleep(100); log.info("--- ReplicatorTestBase::setup completed ---"); @@ -207,11 +262,11 @@ public void setConfig3DefaultValue() { } public void setConfig1DefaultValue(){ - setConfigDefaults(config1, "r1", bkEnsemble1); + setConfigDefaults(config1, cluster1, bkEnsemble1); } public void setConfig2DefaultValue() { - setConfigDefaults(config2, "r2", bkEnsemble2); + setConfigDefaults(config2, cluster2, bkEnsemble2); } private void setConfigDefaults(ServiceConfiguration config, String clusterName, @@ -229,9 +284,16 @@ private void setConfigDefaults(ServiceConfiguration config, String clusterName, config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); - config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH); + config.setTlsCertificateFilePath(brokerCertFilePath); + config.setTlsKeyFilePath(brokerFilePath); + config.setTlsTrustCertsFilePath(caCertFilePath); + config.setTlsEnabledWithKeyStore(tlsWithKeyStore); + config.setTlsKeyStore(brokerKeyStorePath); + config.setTlsKeyStoreType(keyStoreType); + config.setTlsKeyStorePassword(keyStorePassword); + config.setTlsTrustStore(brokerTrustStorePath); + config.setTlsTrustStoreType(keyStoreType); + config.setTlsTrustStorePassword(keyStorePassword); config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); config.setDefaultNumberOfNamespaceBundles(1); config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java index e1506d16188b0..49e4e79539499 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java @@ -21,6 +21,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,10 +52,18 @@ public void cleanup() throws Exception { public void testReplicationClient() throws Exception { log.info("--- Starting ReplicatorTlsTest::testReplicationClient ---"); for (BrokerService ns : List.of(ns1, ns2, ns3)) { + // load the client + ns.getReplicationClient(cluster1, Optional.of(admin1.clusters().getCluster(cluster1))); + ns.getReplicationClient(cluster2, Optional.of(admin1.clusters().getCluster(cluster2))); + ns.getReplicationClient(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); + + // verify the client ns.getReplicationClients().forEach((cluster, client) -> { - assertTrue(((PulsarClientImpl) client).getConfiguration().isUseTls()); - assertEquals(((PulsarClientImpl) client).getConfiguration().getTlsTrustCertsFilePath(), - TLS_SERVER_CERT_FILE_PATH); + ClientConfigurationData configuration = ((PulsarClientImpl) client).getConfiguration(); + assertTrue(configuration.isUseTls()); + assertEquals(configuration.getTlsTrustCertsFilePath(), caCertFilePath); + assertEquals(configuration.getTlsKeyFilePath(), clientKeyFilePath); + assertEquals(configuration.getTlsCertificateFilePath(), clientCertFilePath); }); } } From fb7f14ceb04d612e456b2e5a834385ae3a97f68f Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 22 Feb 2023 00:59:08 -0600 Subject: [PATCH 004/404] [cleanup][broker] Update deprecation warnings to use 3.0.0 (#19586) ### Motivation With https://github.com/apache/pulsar/pull/19573, we should replace deprecation warnings that were set to 2.12.0 with 3.0.0. ### Modifications * Update 4 deprecation warnings. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: This is cosmetic. ### Documentation - [x] `doc` This is a documentation change. ### Matching PR in forked repository PR in forked repository: skipping forked PR since this is a trivial change --- .../pulsar/broker/authentication/AuthenticationProvider.java | 2 +- .../pulsar/broker/authentication/AuthenticationService.java | 2 +- .../broker/authentication/OneStageAuthenticationState.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java index dd1942e318a08..109259537a494 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java @@ -114,7 +114,7 @@ default AuthenticationState newAuthState(AuthData authData, * an {@link AuthenticationDataSource} that was added as the {@link AuthenticatedDataAttributeName} attribute to * the http request. Removing this method removes an unnecessary step in the authentication flow.

*/ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") default AuthenticationState newHttpAuthState(HttpServletRequest request) throws AuthenticationException { return new OneStageAuthenticationState(request, this); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java index e758c2de7e308..d11bb6d76e82e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java @@ -158,7 +158,7 @@ public boolean authenticateHttpRequest(HttpServletRequest request, HttpServletRe /** * @deprecated use {@link #authenticateHttpRequest(HttpServletRequest, HttpServletResponse)} */ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") public String authenticateHttpRequest(HttpServletRequest request, AuthenticationDataSource authData) throws AuthenticationException { String authMethodName = getAuthMethodName(request); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java index 725f24489d5f8..242564a8d1f5a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java @@ -106,7 +106,7 @@ public CompletableFuture authenticateAsync(AuthData authData) { /** * @deprecated use {@link #authenticateAsync(AuthData)} */ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") @Override public AuthData authenticate(AuthData authData) throws AuthenticationException { try { @@ -120,7 +120,7 @@ public AuthData authenticate(AuthData authData) throws AuthenticationException { * @deprecated rely on result from {@link #authenticateAsync(AuthData)}. For more information, see the Javadoc * for {@link AuthenticationState#isComplete()}. */ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") @Override public boolean isComplete() { return authRole != null; From f292bad8ac92eaa0c9f6735b12742c0aa27be5e8 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 22 Feb 2023 11:09:57 -0600 Subject: [PATCH 005/404] [fix][test] ProxyWithAuthorizationTest remove SAN from test certs (#19594) --- .../authentication/tls/broker-cert.pem | 79 ++++++----- .../resources/authentication/tls/cacert.pem | 125 +++++++++--------- .../authentication/tls/client-cert.pem | 79 ++++++----- build/regenerate_certs_for_tests.sh | 16 ++- .../server/ProxyWithAuthorizationTest.java | 38 +++--- .../broker-cacert.pem | 125 +++++++++--------- .../broker-cert.pem | 79 ++++++----- .../client-cacert.pem | 125 +++++++++--------- .../client-cert.pem | 79 ++++++----- .../no-subject-alt-cert.pem | 67 ++++++++++ .../no-subject-alt-key.pem | 28 ++++ .../proxy-cacert.pem | 125 +++++++++--------- .../ProxyWithAuthorizationTest/proxy-cert.pem | 79 ++++++----- .../resources/authentication/tls/cacert.pem | 125 +++++++++--------- .../authentication/tls/client-cert.pem | 79 ++++++----- .../authentication/tls/server-cert.pem | 79 ++++++----- 16 files changed, 718 insertions(+), 609 deletions(-) create mode 100644 pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem create mode 100644 pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem index e9be840d3a083..e2b44e0bf0c42 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:05 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114493 (0xd7a0327703a8fc3d) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = localhost + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:af:bf:b7:2d:98:ad:9d:f6:da:a3:13:d4:62:0f: 98:be:1c:a2:89:22:ba:6f:d5:fd:1f:67:e3:91:03: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 88:1d:a7:42:a1:1c:87:45:4a:e6:5e:aa:9c:7b:71:2e:5c:9e: - 11:85:0f:a3:c5:b4:ea:73:9e:b7:61:9d:4a:e9:cd:1a:c5:2e: - 03:be:a3:2b:b6:12:6a:15:03:04:3f:fb:4a:09:0d:84:0e:dd: - c0:63:2b:0f:13:fb:1f:98:64:49:48:e7:96:d5:41:c4:ca:94: - bf:ab:c5:ea:80:2c:ee:1f:ab:12:54:74:f1:f1:56:ea:03:c0: - 1c:0d:8d:b9:6e:b0:d0:5f:21:c1:d3:e3:45:df:cf:64:69:13: - 6c:54:79:06:7d:53:46:77:3c:21:cc:c4:6a:5f:f9:9a:07:0f: - a5:95:20:f0:0e:93:07:48:96:a9:2c:28:50:21:d7:f8:13:4f: - b8:ca:aa:1f:a6:41:7c:71:1f:ad:11:3f:3d:1e:e9:81:3c:86: - c1:af:2d:39:a0:13:9f:99:ec:9a:47:44:df:28:02:a7:1d:6a: - 8d:c0:1e:24:e8:19:fc:1d:dc:67:29:04:be:0a:d6:c5:81:59: - 27:2c:f5:e5:df:ba:0b:c6:50:e5:b3:bd:73:12:3e:2c:ef:a6: - 8a:ed:eb:86:9a:45:45:52:a3:44:78:12:60:17:e2:3a:32:92: - 03:6e:89:89:16:c5:e0:bc:be:a7:cb:93:4b:d8:56:33:a0:a0: - 53:b2:0d:a5 + 5f:e0:73:7b:5e:db:c0:8b:5e:4c:43:5f:80:94:ca:0b:f8:e9: + 9b:93:91:3d:b1:3a:99:ce:1c:fb:15:32:68:3e:b9:9c:52:d0: + 4b:7f:17:09:ec:af:6b:05:3e:e2:a3:e6:cc:bb:53:d7:ea:4a: + 82:3c:4e:a5:37:ca:f4:1e:38:e2:d6:a5:98:4d:ee:b9:e2:9a: + 48:d2:9f:0a:bc:61:42:70:22:b9:fb:cd:73:72:fb:94:13:ac: + 6e:c5:b6:4b:24:ef:0f:df:2d:e6:56:da:b2:76:e8:16:be:7f: + 3f:1b:99:6e:32:3e:b9:f4:2b:35:72:c7:e4:c6:a5:92:68:c0: + 1f:a0:f7:17:fd:a3:b6:73:98:d3:ea:1c:af:ea:7d:f8:a0:27: + 40:dc:4e:8b:13:28:ba:65:60:c5:90:57:e8:54:c1:83:b4:9d: + f0:ae:2a:de:27:57:e5:a2:e5:f4:87:1c:df:6b:dc:7b:43:ff: + b6:be:0b:3b:b2:8b:1a:36:dc:e3:57:aa:52:ef:23:d6:50:d7: + e4:72:8f:a0:0a:43:de:3d:f2:42:5b:fa:ed:1f:8d:0e:cf:c5: + 6a:ce:3b:8e:fd:6b:68:01:a9:f9:d2:0e:0d:ac:39:8d:f5:6c: + 80:f8:49:af:bb:b9:d4:81:b9:f3:b2:b6:ce:75:1c:20:e8:6a: + 53:dc:26:86 -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgUwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCWxvY2FsaG9zdDCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+/ty2YrZ322qMT1GIPmL4c -ookium/V/R9n45EDmICBDu3Y9nB/LDZoPVPqWDqm1YlmS70eV3ETbUsR5UCldoQk -kkBYgJbJHyzEVeujeXNwXDeaie0vumvjgnxpSgJUi4FePL9MisvqLF6D57cQCF+C -WKOJ0dqSuioo7jAoP1uuEHGWx+ESxbAarURvRDoRSpo8D40GgHs07z9s9F7FRFQe -yN3HgIWA2WjmxlMDd+H+GGEHdwVM7Vm8XUE4au9dobJgmNRIKJUCig79z3sb0hHM -EAxQc9fMOGyD3XkmqpDIm4SGvFnpYmn0mBvEgHh+oBqBndLhZt3EzPxjBKzspzUC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCIHadCoRyHRUrmXqqce3EuXJ4RhQ+jxbTqc563YZ1K6c0axS4DvqMr -thJqFQMEP/tKCQ2EDt3AYysPE/sfmGRJSOeW1UHEypS/q8XqgCzuH6sSVHTx8Vbq -A8AcDY25brDQXyHB0+NF389kaRNsVHkGfVNGdzwhzMRqX/maBw+llSDwDpMHSJap -LChQIdf4E0+4yqofpkF8cR+tET89HumBPIbBry05oBOfmeyaR0TfKAKnHWqNwB4k -6Bn8HdxnKQS+CtbFgVknLPXl37oLxlDls71zEj4s76aK7euGmkVFUqNEeBJgF+I6 -MpIDbomJFsXgvL6ny5NL2FYzoKBTsg2l +MIIDCTCCAfGgAwIBAgIJANegMncDqPw9MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv1f0fZ+OR +A5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICWyR8sxFXr +o3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHakroqKO4w +KD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CFgNlo5sZT +A3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPXzDhsg915 +JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAX+Bz +e17bwIteTENfgJTKC/jpm5ORPbE6mc4c+xUyaD65nFLQS38XCeyvawU+4qPmzLtT +1+pKgjxOpTfK9B444talmE3uueKaSNKfCrxhQnAiufvNc3L7lBOsbsW2SyTvD98t +5lbasnboFr5/PxuZbjI+ufQrNXLH5MalkmjAH6D3F/2jtnOY0+ocr+p9+KAnQNxO +ixMoumVgxZBX6FTBg7Sd8K4q3idX5aLl9Icc32vce0P/tr4LO7KLGjbc41eqUu8j +1lDX5HKPoApD3j3yQlv67R+NDs/Fas47jv1raAGp+dIODaw5jfVsgPhJr7u51IG5 +87K2znUcIOhqU9wmhg== -----END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem index 21bbaba213f69..4ed454ec52a52 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 70:4c:6b:e0:aa:cc:01:77:f2:1f:04:8c:d4:72:03:a5:32:5f:c7:be - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15358526754272834781 (0xd52472b5c5c3f4dd) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:dc:9c:01:30:5f:c5:42:48:10:78:30:5d:66:20: - 0e:74:61:f6:82:74:9f:6f:b2:ed:00:9e:6c:21:b6: - 83:21:6b:54:34:e8:a9:dc:81:83:7a:0e:9f:cc:3d: - eb:97:ee:cf:ca:0e:5f:96:81:dc:e7:75:88:91:2f: - d5:65:74:c2:d8:67:58:d8:41:6a:5f:a9:79:dc:29: - 36:4a:b8:39:20:d2:f8:a8:59:9f:e3:be:f9:61:80: - 1b:ce:63:bb:12:56:06:b9:77:4e:6a:40:65:9b:bf: - 5b:f8:27:88:f5:ff:40:ee:47:bc:2d:8e:c3:a6:62: - 0d:18:76:d1:f5:af:1a:6b:25:4e:d4:55:15:f0:e3: - 97:1b:68:eb:75:b8:80:ea:64:ef:7e:e2:f0:5c:da: - 6d:d6:16:7b:0f:5e:ae:72:47:5a:df:0b:8a:e0:74: - c1:b7:82:0d:97:41:d7:84:16:51:40:37:15:a1:eb: - 70:0c:f1:5a:26:39:11:1e:97:b9:36:32:ce:16:b9: - 42:ad:31:5b:1e:89:f5:3e:07:0e:d6:fc:9a:46:8e: - 87:89:90:5c:f3:00:e4:9b:ce:7b:93:fe:9a:d8:65: - ec:49:5c:e8:eb:41:3d:53:bc:ce:e8:6d:44:ec:76: - 3f:e6:9b:13:e4:f8:d0:1c:00:e6:4f:73:e1:b0:27: - 6f:99 + 00:d0:87:45:0b:b4:83:11:ab:5a:b4:b6:1c:15:d4: + 92:6a:0c:ac:3b:76:da:ff:8d:61:1b:bd:96:bd:d7: + b0:70:23:87:d4:00:19:b2:e5:63:b7:80:58:4a:a4: + d8:a8:a6:4f:eb:c8:8c:54:07:f5:56:52:23:64:fc: + 66:54:39:f1:33:d0:e5:cc:b6:40:c8:d7:9a:9f:0e: + c4:aa:57:b0:b3:e2:41:61:54:ca:1f:90:3b:18:ef: + 60:d2:dc:ee:34:29:33:08:1b:37:4b:c4:ca:7e:cb: + 94:7f:50:c4:8d:16:2f:90:03:94:07:bf:cf:52:ff: + 24:54:56:ac:74:6c:d3:31:8c:ce:ef:b3:14:5a:5b: + 8a:0c:83:2d:e1:f7:4d:60:2f:a1:4d:85:38:96:7f: + 01:2f:9a:99:c7:2e:3d:09:4d:5e:53:df:fd:29:9f: + ff:6b:e4:c2:a1:e3:67:85:db:e2:02:4d:6f:29:d4: + e1:b3:a2:34:71:e0:90:dd:3f:b3:3f:86:41:8c:97: + 09:e6:c3:de:a0:0e:d3:d4:3e:ce:ea:58:70:e6:9f: + 24:a8:19:ca:df:61:b8:9c:c3:4e:53:d0:69:96:44: + 84:76:2b:99:65:08:06:42:d4:b2:76:a7:2f:69:12: + d5:c2:65:a6:ff:2c:77:73:00:e7:97:a5:77:6b:8a: + 9c:3f Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 8B:30:D2:81:7C:BE:AB:4D:76:37:19:2B:69:5E:DB:F7:81:95:73:F5 + A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69 X509v3 Authority Key Identifier: - keyid:8B:30:D2:81:7C:BE:AB:4D:76:37:19:2B:69:5E:DB:F7:81:95:73:F5 + keyid:A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69 + DirName:/CN=CARoot + serial:D5:24:72:B5:C5:C3:F4:DD - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 02:4c:80:4f:a4:b5:f4:70:be:82:cf:3a:ed:40:f9:97:17:22: - 07:5d:e0:9b:4e:54:f8:4b:64:99:f5:07:7f:87:5b:9c:60:ec: - 9f:69:e6:00:97:5a:cd:14:59:31:45:be:b7:bd:c4:ce:57:82: - 1a:4a:62:ce:8e:c8:59:d5:62:43:8b:94:c0:ab:c2:cc:3a:a0: - 69:d3:65:15:82:35:de:85:64:e6:7b:d9:3a:22:12:77:f7:71: - 82:86:d7:6c:e5:69:d5:3a:f2:a7:25:f7:dc:f3:6f:cb:eb:85: - 48:44:63:e2:6d:3c:82:eb:3a:c0:e1:bd:9d:3a:12:11:66:1f: - 05:8f:49:65:31:d6:cf:26:06:46:ba:73:c7:ad:61:fc:14:5f: - 68:d1:ee:02:5f:4b:98:b6:5b:0c:98:4e:61:7b:cb:35:ee:44: - a1:ce:e1:00:a2:56:f0:0d:72:3b:58:66:e8:9a:dc:62:d5:95: - 3e:5a:48:21:a8:7c:f8:1f:5a:13:db:53:33:11:3e:e6:14:39: - cd:2b:3f:77:5b:ee:f7:0c:59:69:2f:46:9a:34:56:89:05:8e: - 40:94:94:3f:95:f6:fa:f9:1a:e8:1a:80:7b:1d:f7:0c:a1:be: - e2:38:98:fd:0f:e7:68:4d:7d:fe:ae:5f:e3:32:c6:5d:37:77: - 7a:28:ce:cc + 21:b1:4d:2b:14:1e:5a:91:5d:28:9e:ba:cb:ed:f1:96:da:c3: + fa:8d:b5:74:e4:c5:fb:2f:3e:39:b4:a6:59:69:dd:84:64:a8: + f0:e0:39:d2:ef:87:cc:8b:09:9f:0a:84:1f:d0:96:9c:4b:64: + ea:08:09:26:1c:84:f4:06:5f:5e:b9:ba:b3:3c:6c:81:e0:93: + 46:89:07:51:95:36:77:96:76:5d:a6:68:71:bb:60:88:a7:83: + 27:7c:66:5d:64:36:cb:8e:bd:02:f7:fb:52:63:83:2f:fe:57: + 4c:d5:0c:1b:ea:ef:88:ad:8c:a9:d4:b3:2c:b8:c4:e2:90:cb: + 0f:24:0e:df:fc:2a:c6:83:08:49:45:b0:41:85:0e:b4:6f:f7: + 18:56:7b:a5:0b:f6:1b:7f:72:88:ee:c8:ef:b3:e3:3e:f0:68: + 1b:c9:55:bb:4d:21:65:6b:9e:5c:dd:60:4b:7f:f1:84:f8:67: + 51:c2:60:88:42:6e:6c:9c:14:b8:96:b0:18:10:97:2c:94:e7: + 79:14:7b:d1:a2:a4:d8:94:84:ac:a9:ca:17:95:c2:27:8b:2b: + d8:19:6a:14:4b:c3:03:a6:30:55:40:bd:ce:0c:c2:d5:af:7d: + 6d:65:89:6b:74:ed:21:12:f1:aa:c9:c9:ba:da:9a:ca:14:6c: + 39:f4:02:32 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUcExr4KrMAXfyHwSM1HIDpTJfx74wDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA3JwBMF/FQkgQeDBdZiAOdGH2gnSfb7LtAJ5sIbaDIWtUNOip3IGD -eg6fzD3rl+7Pyg5floHc53WIkS/VZXTC2GdY2EFqX6l53Ck2Srg5INL4qFmf4775 -YYAbzmO7ElYGuXdOakBlm79b+CeI9f9A7ke8LY7DpmINGHbR9a8aayVO1FUV8OOX -G2jrdbiA6mTvfuLwXNpt1hZ7D16uckda3wuK4HTBt4INl0HXhBZRQDcVoetwDPFa -JjkRHpe5NjLOFrlCrTFbHon1PgcO1vyaRo6HiZBc8wDkm857k/6a2GXsSVzo60E9 -U7zO6G1E7HY/5psT5PjQHADmT3PhsCdvmQIDAQABo1MwUTAdBgNVHQ4EFgQUizDS -gXy+q012NxkraV7b94GVc/UwHwYDVR0jBBgwFoAUizDSgXy+q012NxkraV7b94GV -c/UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAkyAT6S19HC+ -gs867UD5lxciB13gm05U+EtkmfUHf4dbnGDsn2nmAJdazRRZMUW+t73EzleCGkpi -zo7IWdViQ4uUwKvCzDqgadNlFYI13oVk5nvZOiISd/dxgobXbOVp1TrypyX33PNv -y+uFSERj4m08gus6wOG9nToSEWYfBY9JZTHWzyYGRrpzx61h/BRfaNHuAl9LmLZb -DJhOYXvLNe5Eoc7hAKJW8A1yO1hm6JrcYtWVPlpIIah8+B9aE9tTMxE+5hQ5zSs/ -d1vu9wxZaS9GmjRWiQWOQJSUP5X2+vka6BqAex33DKG+4jiY/Q/naE19/q5f4zLG -XTd3eijOzA== +MIIDGjCCAgKgAwIBAgIJANUkcrXFw/TdMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANCH +RQu0gxGrWrS2HBXUkmoMrDt22v+NYRu9lr3XsHAjh9QAGbLlY7eAWEqk2KimT+vI +jFQH9VZSI2T8ZlQ58TPQ5cy2QMjXmp8OxKpXsLPiQWFUyh+QOxjvYNLc7jQpMwgb +N0vEyn7LlH9QxI0WL5ADlAe/z1L/JFRWrHRs0zGMzu+zFFpbigyDLeH3TWAvoU2F +OJZ/AS+amccuPQlNXlPf/Smf/2vkwqHjZ4Xb4gJNbynU4bOiNHHgkN0/sz+GQYyX +CebD3qAO09Q+zupYcOafJKgZyt9huJzDTlPQaZZEhHYrmWUIBkLUsnanL2kS1cJl +pv8sd3MA55eld2uKnD8CAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUp1VrURB1zk5bC2T/qW0j+1eIWWkwQQYDVR0jBDowOIAUp1VrURB1zk5bC2T/ +qW0j+1eIWWmhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANUkcrXFw/TdMA0GCSqG +SIb3DQEBCwUAA4IBAQAhsU0rFB5akV0onrrL7fGW2sP6jbV05MX7Lz45tKZZad2E +ZKjw4DnS74fMiwmfCoQf0JacS2TqCAkmHIT0Bl9eubqzPGyB4JNGiQdRlTZ3lnZd +pmhxu2CIp4MnfGZdZDbLjr0C9/tSY4Mv/ldM1Qwb6u+IrYyp1LMsuMTikMsPJA7f +/CrGgwhJRbBBhQ60b/cYVnulC/Ybf3KI7sjvs+M+8GgbyVW7TSFla55c3WBLf/GE ++GdRwmCIQm5snBS4lrAYEJcslOd5FHvRoqTYlISsqcoXlcIniyvYGWoUS8MDpjBV +QL3ODMLVr31tZYlrdO0hEvGqycm62prKFGw59AIy -----END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem index e5d9e6e74b233..3cf236c401255 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:06 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114494 (0xd7a0327703a8fc3e) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = superUser + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=superUser Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:cd:43:7d:98:40:f9:b0:5b:bc:ae:db:c0:0b:ad: 26:90:96:e0:62:38:ed:68:b1:70:46:3b:de:44:f9: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 90:62:ba:7b:6f:45:95:7a:71:2f:e7:88:0c:64:b8:6c:05:86: - 7f:47:08:ce:d6:e2:5a:32:13:0c:82:ad:a7:af:f0:a2:f7:86: - 79:87:1a:89:78:95:b1:9f:be:c5:8b:39:fd:12:94:b6:e1:69: - ff:fa:1e:c3:82:d8:6c:03:80:45:ac:1c:06:70:bb:77:c3:41: - 5f:b6:9d:fe:36:6f:ae:23:6c:bf:43:79:8e:74:85:8e:96:89: - a9:c4:6d:d9:fa:05:ba:a8:11:7c:82:45:94:3d:9f:b6:7c:2f: - 4e:6d:37:c3:fb:79:7e:0c:d2:15:fa:0e:ea:2d:c9:24:f3:34: - 13:6f:db:d7:55:e1:0c:2f:7e:fe:4c:3b:fa:7e:03:26:0f:6a: - 95:d2:22:ce:27:71:6a:97:ac:36:0a:20:ec:19:a0:78:23:0c: - 54:f3:b1:dd:33:36:7c:b7:61:23:70:8f:7f:c8:5f:e8:9e:b5: - 02:31:4d:b3:40:b0:7b:b2:ee:14:a7:69:22:8b:38:85:5d:04: - 6e:d5:44:41:31:a7:4b:71:86:fb:81:cd:3d:db:96:23:0b:bc: - e1:67:46:0e:87:86:91:4e:1a:35:37:af:a4:ac:9a:de:e3:4f: - 82:47:f1:c4:16:58:11:8f:76:d2:4d:df:a1:c6:a2:8f:33:6d: - 72:15:28:76 + b8:fc:d3:8f:8a:e0:6b:74:57:e2:a3:79:b2:18:60:0b:2c:05: + f9:e3:ae:dd:e9:ad:52:88:52:73:b4:12:b0:39:90:65:12:f5: + 95:0e:5f:4b:f2:06:4a:57:ab:e1:f9:b1:34:68:83:d7:d7:5e: + 69:0a:16:44:ea:1d:97:53:51:10:51:8b:ec:0a:b3:c8:a3:3d: + 85:4d:f4:8f:7d:b3:b5:72:e4:9e:d7:f3:01:bf:66:e1:40:92: + 54:63:16:b6:b5:66:ed:30:38:94:1d:1a:8f:28:34:27:ab:c9: + 5f:d5:16:7e:e4:f5:93:d2:19:35:44:0a:c4:2e:6a:25:38:1d: + ee:5a:c8:29:fa:96:dc:95:82:38:9e:36:3a:68:34:7b:4e:d9: + fa:0d:b2:88:a2:6c:4f:03:18:a7:e3:41:67:38:de:e5:f6:ff: + 2a:1c:f0:ec:1a:02:a7:e8:4e:3a:c3:04:72:f8:6a:4f:28:a6: + cf:0b:a2:db:33:74:d1:10:9e:ec:b4:ac:f8:b1:24:f4:ef:0e: + 05:e4:9d:1b:9a:40:f7:09:66:9c:9d:86:8b:76:96:46:e8:d1: + dc:10:c7:7d:0b:69:41:dc:a7:8e:e3:a3:36:e3:42:63:93:8c: + 91:80:0d:27:11:1c:2d:ae:fb:92:88:6c:6b:09:40:1a:30:dd: + 8f:ac:0f:62 -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgYwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCXN1cGVyVXNlcjCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1DfZhA+bBbvK7bwAutJpCW -4GI47WixcEY73kT5FFGGEOvKkOeI6PmRheDdtbQUuXjjhtVUbWjsFJK0+CJbBT3t -MSVlCAWEyuYMIRJYMscaYKNP0kqeKBl8RYQAjInc3orlT4iRzKTxgUVMfcL/4sGJ -xhJzleI2vduui1poapBR3iuIX6pn9KjjY9y+GYLMnX/mjfuCviIBPVYTO1sEtOjF -GOYuDfq6So3oxlqhUZpKYtev3bT84tXNrplsXGFWC9cMGndc9TpqVLWeM6ypdSia -dq/QelcAG5ETMf1CiCFHBRABL1m7xzrZ4VhMG2xxtpjv3QOCWKMy3JChtqYe4QsC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCQYrp7b0WVenEv54gMZLhsBYZ/RwjO1uJaMhMMgq2nr/Ci94Z5hxqJ -eJWxn77Fizn9EpS24Wn/+h7DgthsA4BFrBwGcLt3w0Fftp3+Nm+uI2y/Q3mOdIWO -lompxG3Z+gW6qBF8gkWUPZ+2fC9ObTfD+3l+DNIV+g7qLckk8zQTb9vXVeEML37+ -TDv6fgMmD2qV0iLOJ3Fql6w2CiDsGaB4IwxU87HdMzZ8t2EjcI9/yF/onrUCMU2z -QLB7su4Up2kiiziFXQRu1URBMadLcYb7gc0925YjC7zhZ0YOh4aRTho1N6+krJre -40+CR/HEFlgRj3bSTd+hxqKPM21yFSh2 +MIIDCTCCAfGgAwIBAgIJANegMncDqPw+MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1osXBGO95E ++RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgFhMrmDCES +WDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5XiNr3brota +aGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36ukqN6MZa +oVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpXABuREzH9 +QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAuPzT +j4rga3RX4qN5shhgCywF+eOu3emtUohSc7QSsDmQZRL1lQ5fS/IGSler4fmxNGiD +19deaQoWROodl1NREFGL7AqzyKM9hU30j32ztXLkntfzAb9m4UCSVGMWtrVm7TA4 +lB0ajyg0J6vJX9UWfuT1k9IZNUQKxC5qJTgd7lrIKfqW3JWCOJ42Omg0e07Z+g2y +iKJsTwMYp+NBZzje5fb/Khzw7BoCp+hOOsMEcvhqTyimzwui2zN00RCe7LSs+LEk +9O8OBeSdG5pA9wlmnJ2Gi3aWRujR3BDHfQtpQdynjuOjNuNCY5OMkYANJxEcLa77 +kohsawlAGjDdj6wPYg== -----END CERTIFICATE----- diff --git a/build/regenerate_certs_for_tests.sh b/build/regenerate_certs_for_tests.sh index fb0274cc19316..fff1c057060f3 100755 --- a/build/regenerate_certs_for_tests.sh +++ b/build/regenerate_certs_for_tests.sh @@ -34,7 +34,16 @@ function reissue_certificate() { keyfile=$1 certfile=$2 openssl x509 -x509toreq -in $certfile -signkey $keyfile -out ${certfile}.csr - openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -out $certfile -days 3650 -CAcreateserial -extfile <(printf "subjectAltName = DNS:localhost, IP:127.0.0.1") + openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -days 3650 -sha256 -CAcreateserial -extfile <(printf "subjectAltName = DNS:localhost, IP:127.0.0.1") > $certfile + rm ${certfile}.csr +} + +function reissue_certificate_no_subject() { + keyfile=$1 + certfile=$2 + openssl x509 -x509toreq -in $certfile -signkey $keyfile -out ${certfile}.csr + openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -days 3650 -sha256 -CAcreateserial > $certfile + rm ${certfile}.csr } generate_ca @@ -54,6 +63,11 @@ cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/Prox reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem \ $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem +# Use $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem as trusted cert +reissue_certificate_no_subject \ + $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem \ + $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem + generate_ca cp ca-cert.pem $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem \ diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index e2362478e4782..de9bb087d3da3 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -68,18 +68,20 @@ public class ProxyWithAuthorizationTest extends ProducerConsumerBase { private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); private final String CLIENT_TOKEN = AuthTokenUtils.createToken(SECRET_KEY, "Client", Optional.empty()); - private final String TLS_PROXY_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem"; + // The Proxy, Client, and SuperUser Client certs are signed by this CA + private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; + + // Proxy and Broker use valid certs that have no Subject Alternative Name to test hostname verification correctly + // fails a connection to an invalid host. + private final String TLS_NO_SUBJECT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem"; + private final String TLS_NO_SUBJECT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem"; private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem"; private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem"; - private final String TLS_BROKER_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem"; - private final String TLS_BROKER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem"; - private final String TLS_BROKER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-key.pem"; private final String TLS_CLIENT_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem"; private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem"; private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-key.pem"; private final String TLS_SUPERUSER_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; private final String TLS_SUPERUSER_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_SUPERUSER_CLIENT_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; private ProxyService proxyService; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); @@ -166,9 +168,9 @@ protected void doInitConf() throws Exception { conf.setBrokerServicePort(Optional.empty()); conf.setWebServicePortTls(Optional.of(0)); conf.setWebServicePort(Optional.empty()); - conf.setTlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_BROKER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(TLS_NO_SUBJECT_CERT_FILE_PATH); + conf.setTlsKeyFilePath(TLS_NO_SUBJECT_KEY_FILE_PATH); conf.setTlsAllowInsecureConnection(false); Set superUserRoles = new HashSet<>(); @@ -177,8 +179,8 @@ protected void doInitConf() throws Exception { conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_BROKER_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_BROKER_KEY_FILE_PATH); - conf.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + "tlsCertFile:" + TLS_SUPERUSER_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); + conf.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setAuthenticationProviders(Set.of(AuthenticationProviderTls.class.getName(), AuthenticationProviderToken.class.getName())); Properties properties = new Properties(); @@ -210,10 +212,10 @@ protected void setup() throws Exception { proxyConfig.setTlsEnabledWithBroker(true); // enable tls and auth&auth at proxy - proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH); - proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH); - proxyConfig.setTlsTrustCertsFilePath(TLS_CLIENT_TRUST_CERT_FILE_PATH); - proxyConfig.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + proxyConfig.setTlsCertificateFilePath(TLS_NO_SUBJECT_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(TLS_NO_SUBJECT_KEY_FILE_PATH); + proxyConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_PROXY_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_PROXY_KEY_FILE_PATH); @@ -446,7 +448,7 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_PROXY_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_PROXY_KEY_FILE_PATH); - proxyConfig.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); @@ -517,7 +519,7 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception .authentication(auth) .tlsKeyFilePath(TLS_CLIENT_KEY_FILE_PATH) .tlsCertificateFilePath(TLS_CLIENT_CERT_FILE_PATH) - .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .operationTimeout(1000, TimeUnit.MILLISECONDS) .build(); @@ -571,7 +573,7 @@ private void createAdminClient() throws Exception { authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .authentication(AuthenticationTls.class.getName(), authParams).build()); } @@ -585,7 +587,7 @@ private PulsarClient createPulsarClient(String proxyServiceUrl, ClientBuilder cl authTls.configure(authParams); return clientBuilder.serviceUrl(proxyServiceUrl).statsInterval(0, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .authentication(authTls).enableTls(true) .operationTimeout(1000, TimeUnit.MILLISECONDS).build(); } diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem index 7d2d58d8d7a06..89de977601909 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 40:cd:a5:a5:35:76:ee:02:57:8b:30:8f:2a:12:34:03:45:c5:96:8c - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15670345994378439095 (0xd97840a8266469b7) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:d8:d5:00:e0:6b:4f:4e:8a:67:08:e9:e3:3f:23: - ef:15:1d:82:10:85:f3:3b:77:9c:96:c1:aa:eb:90: - 41:0b:5b:ae:77:d9:a3:f1:cf:2a:32:40:78:33:6a: - 81:b9:c2:cd:91:36:98:df:41:84:c0:62:8a:a1:03: - 89:8d:2b:b8:91:49:a9:e8:a2:90:ad:b9:cd:23:84: - bc:60:1f:6f:b5:81:9f:9c:cf:d5:26:a8:a5:b6:4d: - 59:5f:5c:7f:da:e8:1d:3d:04:f3:b8:ef:f8:d5:73: - c6:fd:6a:b1:91:ae:16:b7:45:21:9a:1a:1a:76:74: - 01:40:ee:fc:3c:67:be:6a:7f:f4:a3:82:37:ee:43: - 41:f5:67:d5:d5:64:9c:d8:53:75:34:4d:23:80:b5: - 59:13:c2:27:47:8e:20:32:6f:f6:b3:70:bf:5e:15: - 08:7e:d1:bf:aa:4d:06:6b:0d:17:21:eb:95:47:52: - fa:d7:97:ef:1a:5d:63:26:17:36:01:20:ac:57:50: - 34:f0:57:49:38:3d:9c:68:6a:87:91:38:b6:76:9d: - bc:e9:4e:c2:58:54:8d:8a:32:05:9e:ba:cb:f0:d0: - ec:91:67:1d:77:bf:d5:02:77:d4:22:78:94:f4:9a: - 49:fa:ef:b2:9b:30:1a:8a:f0:a7:9a:2b:e5:e9:c7: - 36:c5 + 00:b2:9a:e4:e5:d4:2e:90:21:62:99:07:8a:dd:94: + 92:6a:f7:e9:b7:b5:b4:85:7e:53:04:ff:fa:72:2c: + 77:1b:23:08:c8:91:ff:28:54:67:78:12:40:fc:9e: + bd:be:56:95:8c:c0:97:9f:54:b8:03:06:f3:83:f5: + 14:af:f7:63:1f:51:b9:81:94:08:69:f8:73:ac:1a: + 9a:dc:9b:79:e4:61:36:86:54:5e:b0:4c:5d:6f:6e: + 0f:06:a3:7c:ab:10:43:01:4d:29:21:62:af:dd:b1: + f4:3f:4d:52:39:98:de:09:5b:68:fd:41:2f:00:f2: + 22:94:69:cf:e2:2a:0b:2a:67:29:31:24:f4:77:36: + b9:18:31:97:e6:2a:96:a2:eb:f2:24:c1:fd:89:1a: + f7:51:67:3e:cf:cc:6b:9b:93:3f:9a:19:9b:f2:e4: + b4:cf:b3:99:47:fb:2f:1f:50:e1:de:90:a5:e4:4c: + da:d6:7d:e6:8c:0d:77:84:6c:87:88:99:27:a4:a8: + 9a:7d:58:ac:78:32:0f:6e:8e:0d:2f:78:0d:51:20: + ae:c1:67:2c:f5:25:7a:dd:98:1c:aa:75:3a:f7:87: + 97:a4:38:b9:96:5c:91:47:30:b0:a7:fd:6e:9e:59: + e4:01:5a:e6:e6:b7:f4:01:21:20:2f:9b:54:05:2f: + 46:45 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - DD:AC:A0:40:6E:E9:2B:49:F2:35:DB:B4:E9:98:AD:58:7B:37:6B:55 + 03:72:4A:D9:37:06:FB:B5:C2:04:CF:0B:BF:98:07:FA:C7:6A:85:CE X509v3 Authority Key Identifier: - keyid:DD:AC:A0:40:6E:E9:2B:49:F2:35:DB:B4:E9:98:AD:58:7B:37:6B:55 + keyid:03:72:4A:D9:37:06:FB:B5:C2:04:CF:0B:BF:98:07:FA:C7:6A:85:CE + DirName:/CN=CARoot + serial:D9:78:40:A8:26:64:69:B7 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 07:0c:90:05:fa:2c:c9:4e:05:ec:6b:7d:99:9c:52:2a:20:34: - 46:ac:8d:24:81:f9:a7:f3:1d:03:32:45:82:9a:61:af:1f:63: - 25:6b:97:ca:93:78:e5:d7:87:81:b6:29:22:d4:0d:8d:ed:0e: - bd:85:80:6c:38:e9:86:3c:bd:ee:ff:26:78:0a:f0:a7:54:0b: - af:27:9e:8b:83:b7:10:e9:44:0d:4a:7e:a8:e2:aa:1c:06:f8: - 18:f1:c4:c9:e4:bb:17:41:59:94:b4:dc:78:53:fb:1b:43:57: - 82:59:de:6c:03:52:9a:28:cb:e4:9e:ea:c5:00:93:e0:27:b4: - 4b:e6:b3:c5:88:2d:14:33:10:ff:b0:23:4e:5d:ea:17:97:7d: - f4:e2:c8:fe:c3:4a:77:83:64:ef:c9:b6:3e:77:64:32:07:91: - bd:e1:58:9a:e1:38:ab:eb:d2:e3:cb:05:7c:c7:f3:2b:47:bf: - 36:64:7e:32:5a:62:44:07:c8:8e:9d:55:1a:99:c4:14:5a:66: - ed:5f:8b:ab:dd:eb:36:28:cd:77:47:84:00:ae:a7:34:0e:0d: - 77:df:67:72:08:94:75:52:1b:4a:71:4d:31:5d:aa:1b:aa:b6: - e0:d6:86:52:7c:26:ae:1f:96:ab:06:32:cb:7a:f3:bb:76:3e: - 08:53:9f:64 + 8f:f3:3c:19:a8:82:c9:44:e0:2f:b2:dd:1c:b5:3c:9d:77:2b: + 05:fc:e3:e1:a4:95:3b:c5:7e:d9:c0:c7:51:c5:70:75:f8:e2: + 49:43:8e:78:74:dd:1d:7e:c1:9a:46:12:bd:25:24:59:e4:cd: + 54:3d:1e:b7:93:4f:dc:9b:3c:10:4b:c6:83:b6:cd:a8:36:20: + 79:7e:b7:8c:76:e0:b0:fe:6e:df:2a:8f:97:f8:36:b2:b7:1f: + 8b:7a:60:58:24:46:fe:ba:d7:f1:5b:69:14:53:09:3c:75:72: + ed:ae:10:98:a3:89:bf:0d:5d:16:2e:31:27:90:3c:61:ff:90: + de:cb:68:f9:30:c1:2f:65:a0:93:c3:e2:d0:fc:ca:f2:01:54: + 5c:f8:6e:fc:10:8b:04:c7:0e:4c:81:d7:8e:b0:16:fd:f7:5b: + 4f:fb:12:18:3b:e5:58:61:13:ce:d6:21:33:f7:43:3e:50:26: + b8:ae:37:18:1f:82:ba:76:14:ee:6b:7b:87:67:95:cc:44:55: + b2:8b:aa:af:9f:b5:78:d0:7f:de:f3:7c:91:27:88:95:b5:a6: + 10:05:40:82:57:a7:0e:f4:99:70:c2:e7:af:ea:f2:47:52:84: + 01:78:c0:56:f7:e2:bf:f9:49:b8:1c:ba:4d:e1:2d:f4:28:71: + 78:ae:ac:89 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUQM2lpTV27gJXizCPKhI0A0XFlowwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA2NUA4GtPTopnCOnjPyPvFR2CEIXzO3eclsGq65BBC1uud9mj8c8q -MkB4M2qBucLNkTaY30GEwGKKoQOJjSu4kUmp6KKQrbnNI4S8YB9vtYGfnM/VJqil -tk1ZX1x/2ugdPQTzuO/41XPG/Wqxka4Wt0UhmhoadnQBQO78PGe+an/0o4I37kNB -9WfV1WSc2FN1NE0jgLVZE8InR44gMm/2s3C/XhUIftG/qk0Gaw0XIeuVR1L615fv -Gl1jJhc2ASCsV1A08FdJOD2caGqHkTi2dp286U7CWFSNijIFnrrL8NDskWcdd7/V -AnfUIniU9JpJ+u+ymzAaivCnmivl6cc2xQIDAQABo1MwUTAdBgNVHQ4EFgQU3ayg -QG7pK0nyNdu06ZitWHs3a1UwHwYDVR0jBBgwFoAU3aygQG7pK0nyNdu06ZitWHs3 -a1UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABwyQBfosyU4F -7Gt9mZxSKiA0RqyNJIH5p/MdAzJFgpphrx9jJWuXypN45deHgbYpItQNje0OvYWA -bDjphjy97v8meArwp1QLryeei4O3EOlEDUp+qOKqHAb4GPHEyeS7F0FZlLTceFP7 -G0NXglnebANSmijL5J7qxQCT4Ce0S+azxYgtFDMQ/7AjTl3qF5d99OLI/sNKd4Nk -78m2PndkMgeRveFYmuE4q+vS48sFfMfzK0e/NmR+MlpiRAfIjp1VGpnEFFpm7V+L -q93rNijNd0eEAK6nNA4Nd99ncgiUdVIbSnFNMV2qG6q24NaGUnwmrh+WqwYyy3rz -u3Y+CFOfZA== +MIIDGjCCAgKgAwIBAgIJANl4QKgmZGm3MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKa +5OXULpAhYpkHit2Ukmr36be1tIV+UwT/+nIsdxsjCMiR/yhUZ3gSQPyevb5WlYzA +l59UuAMG84P1FK/3Yx9RuYGUCGn4c6wamtybeeRhNoZUXrBMXW9uDwajfKsQQwFN +KSFir92x9D9NUjmY3glbaP1BLwDyIpRpz+IqCypnKTEk9Hc2uRgxl+YqlqLr8iTB +/Yka91FnPs/Ma5uTP5oZm/LktM+zmUf7Lx9Q4d6QpeRM2tZ95owNd4Rsh4iZJ6So +mn1YrHgyD26ODS94DVEgrsFnLPUlet2YHKp1OveHl6Q4uZZckUcwsKf9bp5Z5AFa +5ua39AEhIC+bVAUvRkUCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUA3JK2TcG+7XCBM8Lv5gH+sdqhc4wQQYDVR0jBDowOIAUA3JK2TcG+7XCBM8L +v5gH+sdqhc6hFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANl4QKgmZGm3MA0GCSqG +SIb3DQEBCwUAA4IBAQCP8zwZqILJROAvst0ctTyddysF/OPhpJU7xX7ZwMdRxXB1 ++OJJQ454dN0dfsGaRhK9JSRZ5M1UPR63k0/cmzwQS8aDts2oNiB5freMduCw/m7f +Ko+X+Daytx+LemBYJEb+utfxW2kUUwk8dXLtrhCYo4m/DV0WLjEnkDxh/5Dey2j5 +MMEvZaCTw+LQ/MryAVRc+G78EIsExw5MgdeOsBb991tP+xIYO+VYYRPO1iEz90M+ +UCa4rjcYH4K6dhTua3uHZ5XMRFWyi6qvn7V40H/e83yRJ4iVtaYQBUCCV6cO9Jlw +wuev6vJHUoQBeMBW9+K/+Um4HLpN4S30KHF4rqyJ -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem index 31743d0684670..8236c0a606d2a 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:07 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114495 (0xd7a0327703a8fc3f) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache Pulsar, OU = Broker, CN = Broker + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Broker, CN=Broker Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:ca:77:dc:2a:13:25:24:cb:29:62:06:12:5f:a8: 92:c9:53:d6:3f:07:ca:aa:0a:5f:72:92:cd:b7:ea: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 8d:1d:69:d2:44:1f:af:68:30:80:c1:91:b2:2f:9a:7e:ca:ff: - 38:46:8e:28:59:02:2d:e7:74:c4:3c:b3:ac:b3:22:53:e9:54: - 3a:e2:4d:4d:65:63:47:dd:38:86:ec:d1:7d:4f:fe:5d:c6:c8: - c8:10:b8:33:5a:4d:9e:83:e3:92:97:c5:f1:d8:e3:97:6d:01: - 50:03:de:25:d8:e4:de:62:70:b8:c4:55:5b:9f:8c:61:b8:d7: - f0:8f:6c:2d:80:cc:b8:7b:8b:b4:54:9a:d6:e1:f9:7f:52:99: - 7b:ef:23:88:61:e5:7c:85:5c:57:98:cc:a6:98:4b:71:84:5c: - ab:5e:82:48:5a:da:5f:d6:84:b5:52:43:df:3c:0f:95:06:29: - 00:94:f8:98:94:6d:1c:c8:76:21:7a:2f:61:34:ab:bd:27:59: - d1:41:99:91:69:68:f7:b6:65:21:e8:9a:b1:9b:ac:72:12:17: - 54:0b:56:08:bd:9d:6b:0e:35:4a:f8:97:b6:83:00:55:96:0c: - 66:13:06:c9:27:5f:cc:d0:81:4b:3e:6e:d2:85:cd:79:7a:8c: - a0:1e:d8:9b:e4:da:e9:ba:51:f1:29:0f:69:00:df:24:a0:55: - 5e:cd:d0:84:c9:4a:a8:b4:12:33:29:6f:8a:8c:d7:a1:b4:8b: - 4a:7d:a2:30 + 0a:35:f9:91:0b:0a:47:88:0e:86:b4:c7:b4:86:9c:b5:6a:e5: + 68:dc:38:f3:5d:f9:ae:15:1c:d9:7a:6a:09:e4:03:f4:d8:71: + 62:1d:c7:e7:ba:0e:d1:00:1a:66:8b:9d:97:6e:b3:c3:99:74: + ad:bf:a9:ab:99:a5:2d:76:d0:87:c5:f5:6e:cd:c3:ef:73:7e: + 23:13:2c:bf:b3:f4:31:93:c2:e9:25:8b:20:de:a7:9b:8a:48: + 32:5f:80:f5:e1:01:4f:14:99:f4:7e:55:62:f9:78:15:18:fa: + 76:a5:0c:88:e5:3d:8a:bf:0f:65:2a:5f:13:5b:c6:03:24:2e: + f6:be:1b:6b:53:f9:93:c2:eb:b6:ee:9d:85:a5:4a:5d:cc:79: + 43:57:9c:47:2b:fc:67:38:de:1d:d5:a3:6a:40:61:df:7e:49: + a8:e0:be:f8:62:dc:b2:86:1f:23:e9:2d:db:0d:8b:4f:e5:05: + 6d:64:6f:11:43:7d:39:e6:68:8f:ee:0a:96:e4:d1:c3:6b:c0: + 55:d7:eb:dc:1c:66:fa:28:d5:1f:92:4d:bb:1c:43:f9:b2:f8: + 4c:36:16:44:58:27:83:32:94:9f:64:d6:bd:f8:d3:fe:c9:e7: + 9d:7b:93:f4:b3:16:61:ad:ff:c3:f3:5d:d3:7b:dc:40:ea:a9: + d1:3d:a7:f5 -----BEGIN CERTIFICATE----- -MIIDETCCAfmgAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgcwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj -aGUgUHVsc2FyMQ8wDQYDVQQLEwZCcm9rZXIxDzANBgNVBAMTBkJyb2tlcjCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMp33CoTJSTLKWIGEl+okslT1j8H -yqoKX3KSzbfqRUdx8GNPWBo9+s6mc5DAqfcl8HZ17bIDF77YilbzT2pMfgNlleVF -641H6GBenjh0UFRloOzYXGVgNBuWg31x1F1/42JZZ+jw1iR9wG43A1RMPQwzOZsz -4VJExUPa6u4s8xwWLkZMfJ9dTW7+jCOe936fOcFxBlL0Jpoi1M/FJTmp0uQkxthK -SKLudiXLPPC/zRB3/4ERQyHMO8wQegeE/MwCokXekS1r0e0XGtBG9K59s4n4MXeV -5UaxqTHW2ONHALKBgduKHNnxzeNNNfY4kQ3qB/CwBk8sTHXCN/81DbFCBgsCAwEA -AaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUA -A4IBAQCNHWnSRB+vaDCAwZGyL5p+yv84Ro4oWQIt53TEPLOssyJT6VQ64k1NZWNH -3TiG7NF9T/5dxsjIELgzWk2eg+OSl8Xx2OOXbQFQA94l2OTeYnC4xFVbn4xhuNfw -j2wtgMy4e4u0VJrW4fl/Upl77yOIYeV8hVxXmMymmEtxhFyrXoJIWtpf1oS1UkPf -PA+VBikAlPiYlG0cyHYhei9hNKu9J1nRQZmRaWj3tmUh6Jqxm6xyEhdUC1YIvZ1r -DjVK+Je2gwBVlgxmEwbJJ1/M0IFLPm7Shc15eoygHtib5NrpulHxKQ9pAN8koFVe -zdCEyUqotBIzKW+KjNehtItKfaIw +MIIDBjCCAe6gAwIBAgIJANegMncDqPw/MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFQxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1bHNhcjEP +MA0GA1UECxMGQnJva2VyMQ8wDQYDVQQDEwZCcm9rZXIwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDKd9wqEyUkyyliBhJfqJLJU9Y/B8qqCl9yks236kVH +cfBjT1gaPfrOpnOQwKn3JfB2de2yAxe+2IpW809qTH4DZZXlReuNR+hgXp44dFBU +ZaDs2FxlYDQbloN9cdRdf+NiWWfo8NYkfcBuNwNUTD0MMzmbM+FSRMVD2uruLPMc +Fi5GTHyfXU1u/owjnvd+nznBcQZS9CaaItTPxSU5qdLkJMbYSkii7nYlyzzwv80Q +d/+BEUMhzDvMEHoHhPzMAqJF3pEta9HtFxrQRvSufbOJ+DF3leVGsakx1tjjRwCy +gYHbihzZ8c3jTTX2OJEN6gfwsAZPLEx1wjf/NQ2xQgYLAgMBAAGjHjAcMBoGA1Ud +EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEACjX5kQsK +R4gOhrTHtIactWrlaNw48135rhUc2XpqCeQD9NhxYh3H57oO0QAaZoudl26zw5l0 +rb+pq5mlLXbQh8X1bs3D73N+IxMsv7P0MZPC6SWLIN6nm4pIMl+A9eEBTxSZ9H5V +Yvl4FRj6dqUMiOU9ir8PZSpfE1vGAyQu9r4ba1P5k8Lrtu6dhaVKXcx5Q1ecRyv8 +ZzjeHdWjakBh335JqOC++GLcsoYfI+kt2w2LT+UFbWRvEUN9OeZoj+4KluTRw2vA +Vdfr3Bxm+ijVH5JNuxxD+bL4TDYWRFgngzKUn2TWvfjT/snnnXuT9LMWYa3/w/Nd +03vcQOqp0T2n9Q== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem index 127f56dd777a5..2a71f0e3afc17 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15559223195710621847 (0xd7ed770f69598897) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: - 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: - 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: - d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: - d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: - 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: - 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: - b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: - 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: - fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: - ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: - 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: - 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: - ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: - 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: - 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: - 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: - 48:9d + 00:bf:3e:e6:db:fd:a2:6c:b9:45:29:e8:d3:90:4d: + 78:29:ef:fc:67:c4:e0:ae:06:fc:f8:2b:cb:3b:4b: + 89:cf:37:3c:ab:86:94:59:c3:50:54:4b:18:d5:8b: + 14:f5:cd:36:58:c6:ac:a1:67:f7:04:58:58:2f:e0: + 89:73:8b:b1:ef:97:a4:16:10:97:e6:6f:2f:18:b9: + 8c:93:7b:7c:5b:4f:8d:09:49:aa:70:59:e5:3b:fa: + c0:b9:4a:ed:14:98:0a:5f:56:b3:49:0a:4d:c0:22: + 1e:75:3d:ba:f9:19:da:68:80:18:ad:b7:8f:de:fd: + 1f:60:33:86:74:46:e0:b7:7a:84:5e:b7:af:5b:57: + 3c:93:ad:37:83:2c:1b:e0:77:a3:84:da:25:1d:16: + 77:3f:25:b1:90:49:28:a6:c8:cf:bc:e4:b9:27:85: + f9:36:4f:47:81:cd:56:26:41:23:10:8f:36:26:ce: + 78:b9:ab:45:ce:9c:eb:a3:2a:11:93:ae:b7:d4:d7: + 57:e9:97:71:5b:fa:ca:0e:71:34:43:87:bb:c0:8e: + 68:f4:4d:c8:64:45:02:5d:81:bd:3b:47:bc:ec:4e: + e4:61:e3:c4:16:f0:ef:83:fd:06:5c:05:50:d8:50: + 41:dc:d6:62:6d:b2:26:7b:d3:6b:f6:da:59:4c:a8: + db:b5 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C X509v3 Authority Key Identifier: - keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + keyid:F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C + DirName:/CN=CARoot + serial:D7:ED:77:0F:69:59:88:97 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: - 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: - 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: - 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: - 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: - ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: - 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: - a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: - a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: - 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: - 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: - 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: - fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: - 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: - ae:fe:19:34 + bc:24:05:65:56:87:f9:87:f4:1b:5a:a3:a0:01:c6:58:c0:7c: + ca:c9:ee:1d:0c:d5:6b:47:28:dc:bd:2f:f1:73:03:e1:a5:08: + 05:56:11:29:d5:48:32:23:d3:7d:46:db:20:58:29:78:19:af: + 6a:a2:d1:b6:6e:30:a0:a3:b1:88:c1:b5:1f:f0:0f:63:5e:ab: + 85:5f:09:0a:3c:20:cb:61:3b:91:80:70:4e:1f:c5:37:34:0d: + 1c:9f:b5:de:93:f1:ae:94:ff:7c:76:7d:6a:a7:19:6f:22:34: + bd:66:07:28:59:f3:60:ef:39:8d:bd:9d:2b:82:08:f0:68:aa: + d6:e0:68:f1:f1:d2:c6:c6:61:88:0d:41:56:40:72:14:14:78: + f9:32:45:de:f9:4d:ef:45:32:a0:21:10:5c:76:f5:ee:fd:a9: + 37:34:04:67:94:72:14:54:5f:27:c3:d3:2d:a6:7f:94:4d:a8: + 98:c4:f9:d9:fc:cd:37:92:3c:c8:bb:2d:a1:f6:ea:14:50:08: + 46:38:9a:cd:71:1b:5b:b4:d9:18:88:77:74:dd:ae:df:b1:58: + 52:f2:8b:e7:bb:0b:0c:92:a6:41:d1:b6:17:64:08:09:7f:7d: + c6:f5:c0:11:a7:29:35:9e:23:9d:e2:08:d5:0e:cb:0c:0c:f2: + d0:7e:38:67 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs -/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 -8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 -AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS -/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO -bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh -Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud -9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU -yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D -VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w -Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX -9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn -eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk -rEibrv4ZNA== +MIIDGjCCAgKgAwIBAgIJANftdw9pWYiXMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8+ +5tv9omy5RSno05BNeCnv/GfE4K4G/PgryztLic83PKuGlFnDUFRLGNWLFPXNNljG +rKFn9wRYWC/giXOLse+XpBYQl+ZvLxi5jJN7fFtPjQlJqnBZ5Tv6wLlK7RSYCl9W +s0kKTcAiHnU9uvkZ2miAGK23j979H2AzhnRG4Ld6hF63r1tXPJOtN4MsG+B3o4Ta +JR0Wdz8lsZBJKKbIz7zkuSeF+TZPR4HNViZBIxCPNibOeLmrRc6c66MqEZOut9TX +V+mXcVv6yg5xNEOHu8COaPRNyGRFAl2BvTtHvOxO5GHjxBbw74P9BlwFUNhQQdzW +Ym2yJnvTa/baWUyo27UCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU+bKtxCRiRzwaWKpm0JES8yDuqWwwQQYDVR0jBDowOIAU+bKtxCRiRzwaWKpm +0JES8yDuqWyhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANftdw9pWYiXMA0GCSqG +SIb3DQEBCwUAA4IBAQC8JAVlVof5h/QbWqOgAcZYwHzKye4dDNVrRyjcvS/xcwPh +pQgFVhEp1UgyI9N9RtsgWCl4Ga9qotG2bjCgo7GIwbUf8A9jXquFXwkKPCDLYTuR +gHBOH8U3NA0cn7Xek/GulP98dn1qpxlvIjS9ZgcoWfNg7zmNvZ0rggjwaKrW4Gjx +8dLGxmGIDUFWQHIUFHj5MkXe+U3vRTKgIRBcdvXu/ak3NARnlHIUVF8nw9Mtpn+U +TaiYxPnZ/M03kjzIuy2h9uoUUAhGOJrNcRtbtNkYiHd03a7fsVhS8ovnuwsMkqZB +0bYXZAgJf33G9cARpyk1niOd4gjVDssMDPLQfjhn -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem index 1a21d9d41387f..6b2387d9a07f8 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:03 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114490 (0xd7a0327703a8fc3a) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache Pulsar, OU = Client, CN = Client + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Client, CN=Client Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:de:1e:10:bd:64:13:c1:6c:7a:49:86:01:3b:ab: ab:1d:ec:b2:93:41:6c:6c:21:f2:e6:15:1b:51:ce: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 8b:88:90:00:1a:15:fa:11:f2:f0:35:6f:0f:f2:76:74:fc:8d: - bc:03:ee:a5:c5:21:17:c9:01:6b:58:93:fa:3e:7b:e0:0d:6d: - db:1f:2a:48:fa:15:34:66:b7:cb:be:82:c6:28:91:99:42:5a: - 36:b6:0b:2f:bb:85:14:88:a9:ea:dd:0a:7a:be:c4:e7:b2:2d: - 82:a9:37:bc:d9:5c:aa:03:2e:54:68:b1:b7:e8:d6:45:a5:8f: - 48:45:2c:9c:7a:55:0a:4a:07:1b:30:8a:49:6d:f4:62:b1:9e: - 92:0e:d9:34:44:6c:6d:e7:a3:18:bb:85:58:6d:da:20:83:d5: - ca:65:63:1e:3b:e6:df:7b:97:40:4f:b1:59:63:a9:b5:80:6f: - 97:51:53:a1:d3:29:1f:1a:26:05:17:59:3e:16:4f:5f:38:36: - 76:30:c6:bf:1e:3e:ed:39:83:91:31:58:01:13:59:5c:c5:e9: - d6:61:e0:f3:5f:c7:47:8a:5f:af:23:98:89:7b:b4:e6:f6:51: - 98:a0:26:31:c8:67:91:6d:d5:68:75:3d:4d:48:44:5f:3b:9c: - df:a7:87:a0:11:02:d2:13:5f:c1:4c:3f:3e:09:59:2e:fc:cb: - c2:c5:f0:f8:91:df:c3:dd:ad:c8:fc:44:23:9b:78:0d:3b:f2: - 82:f6:02:82 + a3:0e:ff:87:38:9a:fe:1c:b7:4b:ac:b1:6c:ad:30:90:94:6e: + 75:36:f6:46:7d:9b:69:1b:0d:92:1b:fc:39:7c:7a:24:fc:4d: + 77:05:8e:70:6e:2e:db:3a:5f:5d:70:80:71:f5:00:7f:6e:12: + 7e:78:58:0b:8f:93:56:64:29:6f:bb:7d:93:a1:fa:2a:83:98: + a3:92:73:df:1d:69:7b:51:00:0f:18:68:a5:75:13:ef:3c:38: + 97:c3:31:84:d6:3c:83:50:77:c3:f6:52:69:5a:9c:35:21:a4: + c2:9f:01:14:b7:2a:a5:3d:71:b2:a6:08:73:10:8e:91:e3:3c: + 69:f9:74:ab:92:f4:16:5d:79:71:3f:3b:58:51:5b:c7:d9:a3: + 85:39:15:60:6b:f7:85:59:e4:4b:96:df:d9:f2:d4:4d:48:34: + 17:50:66:d6:e3:50:49:0d:e7:d5:d9:e0:81:4e:9b:a6:b5:6b: + 72:f3:df:b4:0b:85:5a:e6:e4:ec:28:3a:06:e0:67:e2:be:c1: + 12:7d:9d:5c:ef:3e:77:29:ee:8f:87:44:c7:79:6f:67:0b:fe: + cf:38:76:25:24:be:70:41:99:7f:47:6a:1e:ce:15:f5:bc:4a: + b9:e2:74:15:a2:05:c2:95:02:86:f2:ae:36:a6:88:bc:ad:62: + 3d:07:b8:fd -----BEGIN CERTIFICATE----- -MIIDETCCAfmgAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgMwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj -aGUgUHVsc2FyMQ8wDQYDVQQLEwZDbGllbnQxDzANBgNVBAMTBkNsaWVudDCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN4eEL1kE8FsekmGATurqx3sspNB -bGwh8uYVG1HOrWf9GD5/emSiYl8uC1m07dkXDre8UGZBt+PEcclzcz3YbTSA8uO5 -mI8rVBSVs1Eb1pGFzbc0olC28YZuBzD6rlWgXfl8HJFQYn27FIaSCqwpPigbmcow -Y9ypXwX4OD4wEAKfzJTXR+Aa9Bxolj0SXlghQSzslq2eCFaDepJfS+a9ARZwKK+q -Jx3E/rIJv6W0R9lYS/5BgQ6iRlfBOXyN5LGnJea03fOeJMnnwIwatKvduTO/Ecu+ -uyL3/K3EQEHX7zcIGpVFH9sUXwv4SP9BJMtcjhhITF8Z6bB7ItO8QjJFmtECAwEA -AaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUA -A4IBAQCLiJAAGhX6EfLwNW8P8nZ0/I28A+6lxSEXyQFrWJP6PnvgDW3bHypI+hU0 -ZrfLvoLGKJGZQlo2tgsvu4UUiKnq3Qp6vsTnsi2CqTe82VyqAy5UaLG36NZFpY9I -RSycelUKSgcbMIpJbfRisZ6SDtk0RGxt56MYu4VYbdogg9XKZWMeO+bfe5dAT7FZ -Y6m1gG+XUVOh0ykfGiYFF1k+Fk9fODZ2MMa/Hj7tOYORMVgBE1lcxenWYeDzX8dH -il+vI5iJe7Tm9lGYoCYxyGeRbdVodT1NSERfO5zfp4egEQLSE1/BTD8+CVku/MvC -xfD4kd/D3a3I/EQjm3gNO/KC9gKC +MIIDBjCCAe6gAwIBAgIJANegMncDqPw6MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFQxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1bHNhcjEP +MA0GA1UECxMGQ2xpZW50MQ8wDQYDVQQDEwZDbGllbnQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDeHhC9ZBPBbHpJhgE7q6sd7LKTQWxsIfLmFRtRzq1n +/Rg+f3pkomJfLgtZtO3ZFw63vFBmQbfjxHHJc3M92G00gPLjuZiPK1QUlbNRG9aR +hc23NKJQtvGGbgcw+q5VoF35fByRUGJ9uxSGkgqsKT4oG5nKMGPcqV8F+Dg+MBAC +n8yU10fgGvQcaJY9El5YIUEs7JatnghWg3qSX0vmvQEWcCivqicdxP6yCb+ltEfZ +WEv+QYEOokZXwTl8jeSxpyXmtN3zniTJ58CMGrSr3bkzvxHLvrsi9/ytxEBB1+83 +CBqVRR/bFF8L+Ej/QSTLXI4YSExfGemweyLTvEIyRZrRAgMBAAGjHjAcMBoGA1Ud +EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAow7/hzia +/hy3S6yxbK0wkJRudTb2Rn2baRsNkhv8OXx6JPxNdwWOcG4u2zpfXXCAcfUAf24S +fnhYC4+TVmQpb7t9k6H6KoOYo5Jz3x1pe1EADxhopXUT7zw4l8MxhNY8g1B3w/ZS +aVqcNSGkwp8BFLcqpT1xsqYIcxCOkeM8afl0q5L0Fl15cT87WFFbx9mjhTkVYGv3 +hVnkS5bf2fLUTUg0F1Bm1uNQSQ3n1dnggU6bprVrcvPftAuFWubk7Cg6BuBn4r7B +En2dXO8+dynuj4dEx3lvZwv+zzh2JSS+cEGZf0dqHs4V9bxKueJ0FaIFwpUChvKu +NqaIvK1iPQe4/Q== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem new file mode 100644 index 0000000000000..789a91ca712ce --- /dev/null +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem @@ -0,0 +1,67 @@ +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 15537474201172114492 (0xd7a0327703a8fc3c) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot + Validity + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Broker, CN=Broker + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:ca:77:dc:2a:13:25:24:cb:29:62:06:12:5f:a8: + 92:c9:53:d6:3f:07:ca:aa:0a:5f:72:92:cd:b7:ea: + 45:47:71:f0:63:4f:58:1a:3d:fa:ce:a6:73:90:c0: + a9:f7:25:f0:76:75:ed:b2:03:17:be:d8:8a:56:f3: + 4f:6a:4c:7e:03:65:95:e5:45:eb:8d:47:e8:60:5e: + 9e:38:74:50:54:65:a0:ec:d8:5c:65:60:34:1b:96: + 83:7d:71:d4:5d:7f:e3:62:59:67:e8:f0:d6:24:7d: + c0:6e:37:03:54:4c:3d:0c:33:39:9b:33:e1:52:44: + c5:43:da:ea:ee:2c:f3:1c:16:2e:46:4c:7c:9f:5d: + 4d:6e:fe:8c:23:9e:f7:7e:9f:39:c1:71:06:52:f4: + 26:9a:22:d4:cf:c5:25:39:a9:d2:e4:24:c6:d8:4a: + 48:a2:ee:76:25:cb:3c:f0:bf:cd:10:77:ff:81:11: + 43:21:cc:3b:cc:10:7a:07:84:fc:cc:02:a2:45:de: + 91:2d:6b:d1:ed:17:1a:d0:46:f4:ae:7d:b3:89:f8: + 31:77:95:e5:46:b1:a9:31:d6:d8:e3:47:00:b2:81: + 81:db:8a:1c:d9:f1:cd:e3:4d:35:f6:38:91:0d:ea: + 07:f0:b0:06:4f:2c:4c:75:c2:37:ff:35:0d:b1:42: + 06:0b + Exponent: 65537 (0x10001) + Signature Algorithm: sha256WithRSAEncryption + 67:a9:c5:b1:e0:12:19:67:f7:27:db:87:90:15:29:99:fc:ea: + 62:b3:73:c3:6f:78:fe:50:17:14:8a:61:35:e3:28:ab:3e:c3: + 85:24:ff:70:81:04:0d:b7:7a:eb:e9:dc:06:97:b4:0f:2c:97: + 6d:81:f7:da:dc:f9:ff:91:94:69:5c:15:29:3a:25:87:ff:ef: + 98:6d:5a:36:19:2d:10:cf:d8:3a:d4:45:30:75:5c:52:58:ef: + e6:6c:27:a0:17:a1:a6:76:05:f4:f3:cb:89:89:61:32:c5:bf: + f0:f3:1c:85:90:78:88:c4:37:63:5c:e6:39:43:c0:b0:51:9d: + cc:51:9f:32:b3:78:47:3e:5e:da:58:12:72:df:ba:11:17:a5: + 40:b8:ef:9e:e1:40:49:38:51:7e:76:1e:6c:7f:d3:70:02:de: + af:bb:a6:e0:53:d8:1d:2e:e5:b6:98:6c:27:92:cf:86:3d:0f: + 01:13:95:5d:40:35:47:dc:1b:4c:e9:52:5e:34:98:13:35:35: + 5b:c4:df:fd:61:99:d4:7f:f4:04:fb:26:97:d7:25:8e:fc:1a: + 13:88:37:53:b2:91:3d:0f:0d:9c:31:8a:d0:76:31:dd:50:85: + 43:8a:9b:46:20:a8:3f:f7:9c:30:bb:39:cc:02:ef:7e:22:32: + 8a:df:7d:93 +-----BEGIN CERTIFICATE----- +MIIC4TCCAckCCQDXoDJ3A6j8PDANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZD +QVJvb3QwHhcNMjMwMjIyMDYyNjMyWhcNMzMwMjE5MDYyNjMyWjBUMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAoTDUFwYWNoZSBQdWxzYXIxDzANBgNV +BAsTBkJyb2tlcjEPMA0GA1UEAxMGQnJva2VyMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAynfcKhMlJMspYgYSX6iSyVPWPwfKqgpfcpLNt+pFR3HwY09Y +Gj36zqZzkMCp9yXwdnXtsgMXvtiKVvNPakx+A2WV5UXrjUfoYF6eOHRQVGWg7Nhc +ZWA0G5aDfXHUXX/jYlln6PDWJH3AbjcDVEw9DDM5mzPhUkTFQ9rq7izzHBYuRkx8 +n11Nbv6MI573fp85wXEGUvQmmiLUz8UlOanS5CTG2EpIou52Jcs88L/NEHf/gRFD +Icw7zBB6B4T8zAKiRd6RLWvR7Rca0Eb0rn2zifgxd5XlRrGpMdbY40cAsoGB24oc +2fHN40019jiRDeoH8LAGTyxMdcI3/zUNsUIGCwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQBnqcWx4BIZZ/cn24eQFSmZ/Opis3PDb3j+UBcUimE14yirPsOFJP9wgQQN +t3rr6dwGl7QPLJdtgffa3Pn/kZRpXBUpOiWH/++YbVo2GS0Qz9g61EUwdVxSWO/m +bCegF6GmdgX088uJiWEyxb/w8xyFkHiIxDdjXOY5Q8CwUZ3MUZ8ys3hHPl7aWBJy +37oRF6VAuO+e4UBJOFF+dh5sf9NwAt6vu6bgU9gdLuW2mGwnks+GPQ8BE5VdQDVH +3BtM6VJeNJgTNTVbxN/9YZnUf/QE+yaX1yWO/BoTiDdTspE9Dw2cMYrQdjHdUIVD +iptGIKg/95wwuznMAu9+IjKK332T +-----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem new file mode 100644 index 0000000000000..63bbb7bfea469 --- /dev/null +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDKd9wqEyUkyyli +BhJfqJLJU9Y/B8qqCl9yks236kVHcfBjT1gaPfrOpnOQwKn3JfB2de2yAxe+2IpW +809qTH4DZZXlReuNR+hgXp44dFBUZaDs2FxlYDQbloN9cdRdf+NiWWfo8NYkfcBu +NwNUTD0MMzmbM+FSRMVD2uruLPMcFi5GTHyfXU1u/owjnvd+nznBcQZS9CaaItTP +xSU5qdLkJMbYSkii7nYlyzzwv80Qd/+BEUMhzDvMEHoHhPzMAqJF3pEta9HtFxrQ +RvSufbOJ+DF3leVGsakx1tjjRwCygYHbihzZ8c3jTTX2OJEN6gfwsAZPLEx1wjf/ +NQ2xQgYLAgMBAAECggEARpLZD2F1BQo79osfRHDCGaM7fuT8Y6ER/CHnyz/BvlGc +9UDm+N652eZzSfWeSSPUWbZpkC87y643Km/NMsRO+Ggkg7KHlMuH2G+ivxLsHT7/ +hQ81xbBu+V7Rnpxa5ex6GgIIEk5Alp+uv7w1UODyNpp0bgD7fW2zRR+93B+W7ia+ +aWLcFur1LgGUVpqmlDKZBLD+q3oJ7ddi/uam8WS41IxtUvUVW4L8Pz4sCGjVqEMC +1SbUuuNT5dWLas21c5RhLn1mfyKzLSfeL63+WLuaEobR3GpLDJeG/P6CUCJfrN+j +NtTDFq89QxGzgN6Rvy9MuHC4kHWHvgGlfZ7uZdzWgQKBgQDl3CabW+ZNPCZk3JHU +fGI0Xb3jQElooXOqZOH+FgGKnrbNb7j04Gjs1P4/XibnVsvgwCL8TbR1hgBD6/Qx +z0Sd2T0nwCmLyO9LzyOrlpcKaKF+4OYFPKiqZGV1jXhCQXH9b7IXufS8U4uXwD+Z +elw5MOD6DON7ud9V5E/J5ST58QKBgQDhfkKvtgzaLPD17Bx0M30buHzQuQHplpc4 +J0WGWUXR6rui5tCeHoASAl+UNAFReWJ7Ra+iTHMNqwolVsSQVzmX6e8342f9y0bV +3iv1ge/dA75gEqxifqSXHVm6T/j40DBIr4fwjl5L2qCB/JKCyRvoCK3pDrYZLXWP +DRWhssujuwKBgQCfQBhrWI9FgV/kT0Clo4tyVmQBtv9lAz6clgpQvDRTMsTZrgbJ +eVSYiLSheHyhmGvmCZfzj25wYed7J1Vm0P/sEJ8jFCp0k0DfF+LRtaJtbrI8sloK +1MzSSH5WpC3mUWtFOAZ+E7Kwa31yJJqrna+ZW/jypM1SYiOOYYC6Ewy8MQKBgFdq +GPQBAQ57KZZMR+OMKk3awRgxAFrLdCfioYMpjHWKJ99I10rUzBUvMlpDptcs1U6w +fxvNwzRjP/Wlo2HJTpxjpcbms2Ohr/4suKHeE1x8nQqlcopkSe4DBMvDQOND4dPr +qClLJ6cERADgJvPofpb+9lxIxbMQ+mfQTLh4lZUNAoGAPfAhkt8i6L3VkBIMaV9X +U+6q4brsT0dNLOO/lgf5FXQuCg0WIgBIb1vrGDD1i9WAUiNN8zzYK9UxqjpAtRAe +LgPYX5GHXR0ceR0MQNHdbc4RRjJbPmgey+d7pc9EUn8WWt/uXIeo01DHBPPjsgHr +k/JZjqmRla+2pklmoG2sfI0= +-----END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem index 127f56dd777a5..2a71f0e3afc17 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15559223195710621847 (0xd7ed770f69598897) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: - 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: - 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: - d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: - d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: - 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: - 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: - b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: - 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: - fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: - ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: - 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: - 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: - ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: - 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: - 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: - 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: - 48:9d + 00:bf:3e:e6:db:fd:a2:6c:b9:45:29:e8:d3:90:4d: + 78:29:ef:fc:67:c4:e0:ae:06:fc:f8:2b:cb:3b:4b: + 89:cf:37:3c:ab:86:94:59:c3:50:54:4b:18:d5:8b: + 14:f5:cd:36:58:c6:ac:a1:67:f7:04:58:58:2f:e0: + 89:73:8b:b1:ef:97:a4:16:10:97:e6:6f:2f:18:b9: + 8c:93:7b:7c:5b:4f:8d:09:49:aa:70:59:e5:3b:fa: + c0:b9:4a:ed:14:98:0a:5f:56:b3:49:0a:4d:c0:22: + 1e:75:3d:ba:f9:19:da:68:80:18:ad:b7:8f:de:fd: + 1f:60:33:86:74:46:e0:b7:7a:84:5e:b7:af:5b:57: + 3c:93:ad:37:83:2c:1b:e0:77:a3:84:da:25:1d:16: + 77:3f:25:b1:90:49:28:a6:c8:cf:bc:e4:b9:27:85: + f9:36:4f:47:81:cd:56:26:41:23:10:8f:36:26:ce: + 78:b9:ab:45:ce:9c:eb:a3:2a:11:93:ae:b7:d4:d7: + 57:e9:97:71:5b:fa:ca:0e:71:34:43:87:bb:c0:8e: + 68:f4:4d:c8:64:45:02:5d:81:bd:3b:47:bc:ec:4e: + e4:61:e3:c4:16:f0:ef:83:fd:06:5c:05:50:d8:50: + 41:dc:d6:62:6d:b2:26:7b:d3:6b:f6:da:59:4c:a8: + db:b5 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C X509v3 Authority Key Identifier: - keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + keyid:F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C + DirName:/CN=CARoot + serial:D7:ED:77:0F:69:59:88:97 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: - 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: - 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: - 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: - 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: - ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: - 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: - a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: - a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: - 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: - 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: - 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: - fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: - 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: - ae:fe:19:34 + bc:24:05:65:56:87:f9:87:f4:1b:5a:a3:a0:01:c6:58:c0:7c: + ca:c9:ee:1d:0c:d5:6b:47:28:dc:bd:2f:f1:73:03:e1:a5:08: + 05:56:11:29:d5:48:32:23:d3:7d:46:db:20:58:29:78:19:af: + 6a:a2:d1:b6:6e:30:a0:a3:b1:88:c1:b5:1f:f0:0f:63:5e:ab: + 85:5f:09:0a:3c:20:cb:61:3b:91:80:70:4e:1f:c5:37:34:0d: + 1c:9f:b5:de:93:f1:ae:94:ff:7c:76:7d:6a:a7:19:6f:22:34: + bd:66:07:28:59:f3:60:ef:39:8d:bd:9d:2b:82:08:f0:68:aa: + d6:e0:68:f1:f1:d2:c6:c6:61:88:0d:41:56:40:72:14:14:78: + f9:32:45:de:f9:4d:ef:45:32:a0:21:10:5c:76:f5:ee:fd:a9: + 37:34:04:67:94:72:14:54:5f:27:c3:d3:2d:a6:7f:94:4d:a8: + 98:c4:f9:d9:fc:cd:37:92:3c:c8:bb:2d:a1:f6:ea:14:50:08: + 46:38:9a:cd:71:1b:5b:b4:d9:18:88:77:74:dd:ae:df:b1:58: + 52:f2:8b:e7:bb:0b:0c:92:a6:41:d1:b6:17:64:08:09:7f:7d: + c6:f5:c0:11:a7:29:35:9e:23:9d:e2:08:d5:0e:cb:0c:0c:f2: + d0:7e:38:67 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs -/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 -8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 -AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS -/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO -bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh -Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud -9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU -yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D -VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w -Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX -9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn -eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk -rEibrv4ZNA== +MIIDGjCCAgKgAwIBAgIJANftdw9pWYiXMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8+ +5tv9omy5RSno05BNeCnv/GfE4K4G/PgryztLic83PKuGlFnDUFRLGNWLFPXNNljG +rKFn9wRYWC/giXOLse+XpBYQl+ZvLxi5jJN7fFtPjQlJqnBZ5Tv6wLlK7RSYCl9W +s0kKTcAiHnU9uvkZ2miAGK23j979H2AzhnRG4Ld6hF63r1tXPJOtN4MsG+B3o4Ta +JR0Wdz8lsZBJKKbIz7zkuSeF+TZPR4HNViZBIxCPNibOeLmrRc6c66MqEZOut9TX +V+mXcVv6yg5xNEOHu8COaPRNyGRFAl2BvTtHvOxO5GHjxBbw74P9BlwFUNhQQdzW +Ym2yJnvTa/baWUyo27UCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU+bKtxCRiRzwaWKpm0JES8yDuqWwwQQYDVR0jBDowOIAU+bKtxCRiRzwaWKpm +0JES8yDuqWyhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANftdw9pWYiXMA0GCSqG +SIb3DQEBCwUAA4IBAQC8JAVlVof5h/QbWqOgAcZYwHzKye4dDNVrRyjcvS/xcwPh +pQgFVhEp1UgyI9N9RtsgWCl4Ga9qotG2bjCgo7GIwbUf8A9jXquFXwkKPCDLYTuR +gHBOH8U3NA0cn7Xek/GulP98dn1qpxlvIjS9ZgcoWfNg7zmNvZ0rggjwaKrW4Gjx +8dLGxmGIDUFWQHIUFHj5MkXe+U3vRTKgIRBcdvXu/ak3NARnlHIUVF8nw9Mtpn+U +TaiYxPnZ/M03kjzIuy2h9uoUUAhGOJrNcRtbtNkYiHd03a7fsVhS8ovnuwsMkqZB +0bYXZAgJf33G9cARpyk1niOd4gjVDssMDPLQfjhn -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem index e2c1e5a230c26..8b0624c0b7f3e 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:04 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114491 (0xd7a0327703a8fc3b) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache Pulsar, OU = Proxy, CN = Proxy + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Proxy, CN=Proxy Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:c3:5c:c5:ad:17:dc:f4:d4:c4:ea:1c:60:5a:24: 46:13:d9:cf:c0:cd:83:2e:2f:82:70:e5:e0:8d:33: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 8d:b6:2c:5f:87:13:06:a8:66:ce:11:2a:2c:20:1e:c7:ee:50: - 75:a7:d1:7c:ad:c6:ec:d1:18:d0:fa:aa:00:fa:08:f9:0f:cc: - df:59:9a:6b:1c:18:07:15:84:d0:9a:24:8d:dd:46:79:9c:dc: - 9e:3e:97:10:24:b2:9d:d4:f6:c5:79:58:87:7c:a6:af:cf:69: - 23:fb:43:7a:0f:4d:26:e0:e9:66:c5:ad:fa:88:e2:c5:6e:6a: - ce:70:0c:8f:73:01:d6:fd:a9:1f:31:49:41:17:45:22:cc:a6: - 71:e4:f4:0f:0f:2e:3e:49:0b:5f:04:94:36:49:fa:72:42:c9: - 25:75:84:9a:dc:16:cb:69:44:44:e5:3a:ff:26:f6:44:42:4c: - 6c:e2:56:d6:3e:bc:f2:8b:83:de:e2:91:70:65:b9:d0:dd:a3: - d1:de:53:27:77:13:2d:86:27:c3:40:2f:c1:a5:50:1c:5a:44: - 51:b4:29:11:c3:30:9d:1a:96:25:7a:d6:05:70:ad:06:0d:f2: - 9b:b1:b6:82:39:06:c7:7c:b2:49:04:19:e4:7e:87:b8:d8:42: - 1d:ab:ed:d0:b0:7f:79:6b:89:75:2f:6a:26:67:3d:33:57:5f: - 5a:49:52:98:3b:2a:e5:43:d7:f9:97:ca:75:cd:6f:e9:e4:66: - b6:d6:c2:c7 + 93:72:f7:a5:ba:a1:ba:0f:50:d4:cd:c4:63:a6:5d:1f:9e:62: + 8c:87:45:05:78:f7:27:c2:e3:1c:b3:eb:a2:00:88:f4:77:4d: + 9e:f8:cc:26:0c:fd:03:56:fc:d3:23:59:93:69:46:6f:72:94: + 4f:b9:ba:2f:d5:66:f8:ed:00:89:e9:e7:87:fa:0e:5a:9d:1b: + b9:f7:0f:dd:c6:ab:83:1d:f9:5d:1a:8a:f9:0e:34:f4:85:2c: + 69:cd:37:44:93:ab:6b:4b:14:0f:a2:72:56:0b:82:60:47:82: + 50:ba:5c:f6:2f:3d:da:f8:42:40:39:f8:4a:bf:f1:30:ec:7f: + bd:5d:ce:4a:0e:15:3b:ca:d8:12:8e:da:58:f0:d5:b2:a0:82: + d2:2b:60:21:10:57:a5:73:30:4c:67:82:32:fe:2a:d4:b1:87: + cf:33:bb:9e:c4:9c:2d:ce:99:d5:9f:9f:30:3e:5f:f4:49:40: + d0:1a:6e:90:ab:88:d6:c3:f6:36:11:52:e1:97:ea:3a:ce:ee: + 49:04:57:6d:5c:6f:a6:ca:21:21:7e:4d:0a:b6:7a:c5:14:f2: + 39:00:59:de:88:30:74:f2:46:a8:95:a4:19:a4:0f:bd:b5:b0: + f1:41:c8:8b:70:c7:67:a8:d0:b2:3b:21:4f:73:fd:16:e6:1d: + a0:71:b0:33 -----BEGIN CERTIFICATE----- -MIIDDzCCAfegAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgQwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj -aGUgUHVsc2FyMQ4wDAYDVQQLEwVQcm94eTEOMAwGA1UEAxMFUHJveHkwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDXMWtF9z01MTqHGBaJEYT2c/AzYMu -L4Jw5eCNM72Vtc/G8FTVjb2HDWJsHT9SZnT/BjMcPNXtLmPZlsbxmILHlEq8ZPKb -OlTsgZm8FIJDhwxr2gOMqgtB1/4nxPmIgTSx/yrgbdBH3cERpVSpUzLNj/Z1WI4F -5NmxrGn+tlTDrTYEonf1U7Z0g9VqAeCWtaKvUI+1152nwr34MYYJX3wKsts04YAl -F199b4vcjtX5z8/19o9q/j6WAMlWsNDjRt65popem45/6hnMolt1IjwdNkjk8hoB -lWHB8HonnYOWdMypBEIIUzSYLrfjg/nyoynhI8TtoBz2Ku3cwN+XqfONAgMBAAGj -HjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOC -AQEAjbYsX4cTBqhmzhEqLCAex+5QdafRfK3G7NEY0PqqAPoI+Q/M31maaxwYBxWE -0Jokjd1GeZzcnj6XECSyndT2xXlYh3ymr89pI/tDeg9NJuDpZsWt+ojixW5qznAM -j3MB1v2pHzFJQRdFIsymceT0Dw8uPkkLXwSUNkn6ckLJJXWEmtwWy2lEROU6/yb2 -REJMbOJW1j688ouD3uKRcGW50N2j0d5TJ3cTLYYnw0AvwaVQHFpEUbQpEcMwnRqW -JXrWBXCtBg3ym7G2gjkGx3yySQQZ5H6HuNhCHavt0LB/eWuJdS9qJmc9M1dfWklS -mDsq5UPX+ZfKdc1v6eRmttbCxw== +MIIDBDCCAeygAwIBAgIJANegMncDqPw7MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFIxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1bHNhcjEO +MAwGA1UECxMFUHJveHkxDjAMBgNVBAMTBVByb3h5MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAw1zFrRfc9NTE6hxgWiRGE9nPwM2DLi+CcOXgjTO9lbXP +xvBU1Y29hw1ibB0/UmZ0/wYzHDzV7S5j2ZbG8ZiCx5RKvGTymzpU7IGZvBSCQ4cM +a9oDjKoLQdf+J8T5iIE0sf8q4G3QR93BEaVUqVMyzY/2dViOBeTZsaxp/rZUw602 +BKJ39VO2dIPVagHglrWir1CPtdedp8K9+DGGCV98CrLbNOGAJRdffW+L3I7V+c/P +9faPav4+lgDJVrDQ40beuaaKXpuOf+oZzKJbdSI8HTZI5PIaAZVhwfB6J52DlnTM +qQRCCFM0mC6344P58qMp4SPE7aAc9irt3MDfl6nzjQIDAQABox4wHDAaBgNVHREE +EzARgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggEBAJNy96W6oboP +UNTNxGOmXR+eYoyHRQV49yfC4xyz66IAiPR3TZ74zCYM/QNW/NMjWZNpRm9ylE+5 +ui/VZvjtAInp54f6DlqdG7n3D93Gq4Md+V0aivkONPSFLGnNN0STq2tLFA+iclYL +gmBHglC6XPYvPdr4QkA5+Eq/8TDsf71dzkoOFTvK2BKO2ljw1bKggtIrYCEQV6Vz +MExngjL+KtSxh88zu57EnC3OmdWfnzA+X/RJQNAabpCriNbD9jYRUuGX6jrO7kkE +V21cb6bKISF+TQq2esUU8jkAWd6IMHTyRqiVpBmkD721sPFByItwx2eo0LI7IU9z +/RbmHaBxsDM= -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem index 127f56dd777a5..2a71f0e3afc17 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15559223195710621847 (0xd7ed770f69598897) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: - 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: - 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: - d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: - d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: - 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: - 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: - b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: - 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: - fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: - ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: - 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: - 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: - ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: - 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: - 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: - 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: - 48:9d + 00:bf:3e:e6:db:fd:a2:6c:b9:45:29:e8:d3:90:4d: + 78:29:ef:fc:67:c4:e0:ae:06:fc:f8:2b:cb:3b:4b: + 89:cf:37:3c:ab:86:94:59:c3:50:54:4b:18:d5:8b: + 14:f5:cd:36:58:c6:ac:a1:67:f7:04:58:58:2f:e0: + 89:73:8b:b1:ef:97:a4:16:10:97:e6:6f:2f:18:b9: + 8c:93:7b:7c:5b:4f:8d:09:49:aa:70:59:e5:3b:fa: + c0:b9:4a:ed:14:98:0a:5f:56:b3:49:0a:4d:c0:22: + 1e:75:3d:ba:f9:19:da:68:80:18:ad:b7:8f:de:fd: + 1f:60:33:86:74:46:e0:b7:7a:84:5e:b7:af:5b:57: + 3c:93:ad:37:83:2c:1b:e0:77:a3:84:da:25:1d:16: + 77:3f:25:b1:90:49:28:a6:c8:cf:bc:e4:b9:27:85: + f9:36:4f:47:81:cd:56:26:41:23:10:8f:36:26:ce: + 78:b9:ab:45:ce:9c:eb:a3:2a:11:93:ae:b7:d4:d7: + 57:e9:97:71:5b:fa:ca:0e:71:34:43:87:bb:c0:8e: + 68:f4:4d:c8:64:45:02:5d:81:bd:3b:47:bc:ec:4e: + e4:61:e3:c4:16:f0:ef:83:fd:06:5c:05:50:d8:50: + 41:dc:d6:62:6d:b2:26:7b:d3:6b:f6:da:59:4c:a8: + db:b5 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C X509v3 Authority Key Identifier: - keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + keyid:F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C + DirName:/CN=CARoot + serial:D7:ED:77:0F:69:59:88:97 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: - 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: - 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: - 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: - 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: - ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: - 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: - a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: - a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: - 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: - 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: - 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: - fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: - 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: - ae:fe:19:34 + bc:24:05:65:56:87:f9:87:f4:1b:5a:a3:a0:01:c6:58:c0:7c: + ca:c9:ee:1d:0c:d5:6b:47:28:dc:bd:2f:f1:73:03:e1:a5:08: + 05:56:11:29:d5:48:32:23:d3:7d:46:db:20:58:29:78:19:af: + 6a:a2:d1:b6:6e:30:a0:a3:b1:88:c1:b5:1f:f0:0f:63:5e:ab: + 85:5f:09:0a:3c:20:cb:61:3b:91:80:70:4e:1f:c5:37:34:0d: + 1c:9f:b5:de:93:f1:ae:94:ff:7c:76:7d:6a:a7:19:6f:22:34: + bd:66:07:28:59:f3:60:ef:39:8d:bd:9d:2b:82:08:f0:68:aa: + d6:e0:68:f1:f1:d2:c6:c6:61:88:0d:41:56:40:72:14:14:78: + f9:32:45:de:f9:4d:ef:45:32:a0:21:10:5c:76:f5:ee:fd:a9: + 37:34:04:67:94:72:14:54:5f:27:c3:d3:2d:a6:7f:94:4d:a8: + 98:c4:f9:d9:fc:cd:37:92:3c:c8:bb:2d:a1:f6:ea:14:50:08: + 46:38:9a:cd:71:1b:5b:b4:d9:18:88:77:74:dd:ae:df:b1:58: + 52:f2:8b:e7:bb:0b:0c:92:a6:41:d1:b6:17:64:08:09:7f:7d: + c6:f5:c0:11:a7:29:35:9e:23:9d:e2:08:d5:0e:cb:0c:0c:f2: + d0:7e:38:67 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs -/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 -8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 -AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS -/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO -bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh -Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud -9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU -yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D -VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w -Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX -9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn -eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk -rEibrv4ZNA== +MIIDGjCCAgKgAwIBAgIJANftdw9pWYiXMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8+ +5tv9omy5RSno05BNeCnv/GfE4K4G/PgryztLic83PKuGlFnDUFRLGNWLFPXNNljG +rKFn9wRYWC/giXOLse+XpBYQl+ZvLxi5jJN7fFtPjQlJqnBZ5Tv6wLlK7RSYCl9W +s0kKTcAiHnU9uvkZ2miAGK23j979H2AzhnRG4Ld6hF63r1tXPJOtN4MsG+B3o4Ta +JR0Wdz8lsZBJKKbIz7zkuSeF+TZPR4HNViZBIxCPNibOeLmrRc6c66MqEZOut9TX +V+mXcVv6yg5xNEOHu8COaPRNyGRFAl2BvTtHvOxO5GHjxBbw74P9BlwFUNhQQdzW +Ym2yJnvTa/baWUyo27UCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU+bKtxCRiRzwaWKpm0JES8yDuqWwwQQYDVR0jBDowOIAU+bKtxCRiRzwaWKpm +0JES8yDuqWyhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANftdw9pWYiXMA0GCSqG +SIb3DQEBCwUAA4IBAQC8JAVlVof5h/QbWqOgAcZYwHzKye4dDNVrRyjcvS/xcwPh +pQgFVhEp1UgyI9N9RtsgWCl4Ga9qotG2bjCgo7GIwbUf8A9jXquFXwkKPCDLYTuR +gHBOH8U3NA0cn7Xek/GulP98dn1qpxlvIjS9ZgcoWfNg7zmNvZ0rggjwaKrW4Gjx +8dLGxmGIDUFWQHIUFHj5MkXe+U3vRTKgIRBcdvXu/ak3NARnlHIUVF8nw9Mtpn+U +TaiYxPnZ/M03kjzIuy2h9uoUUAhGOJrNcRtbtNkYiHd03a7fsVhS8ovnuwsMkqZB +0bYXZAgJf33G9cARpyk1niOd4gjVDssMDPLQfjhn -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem index 192d686246f1a..12f8d1fcea0ac 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:01 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114488 (0xd7a0327703a8fc38) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = superUser + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=superUser Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:cd:43:7d:98:40:f9:b0:5b:bc:ae:db:c0:0b:ad: 26:90:96:e0:62:38:ed:68:b1:70:46:3b:de:44:f9: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 96:c2:23:2d:46:d0:3d:23:0e:ab:3d:b6:1e:31:96:00:eb:ae: - 17:ac:6e:c0:d4:1a:8d:0f:36:63:27:02:49:4e:24:cf:d3:80: - 88:3a:4f:d0:f1:e5:1c:df:2d:8a:ab:ae:8d:48:77:a0:d0:dc: - d5:80:1c:a1:3d:0d:49:64:bf:cb:39:84:c9:f3:5d:e0:2d:ba: - a0:f2:ac:03:85:44:a1:97:6b:0b:de:ed:a7:49:19:46:b2:18: - 49:21:62:43:52:36:6f:47:6c:21:6b:5e:41:85:28:71:6c:22: - 27:35:76:82:ed:ac:ad:d7:fa:9d:4c:7d:6f:44:7e:06:dd:8a: - 11:32:0c:d9:d0:f6:63:2a:40:ae:0d:5a:df:9e:d7:91:8a:db: - 2d:95:f3:19:f0:8f:1e:34:e3:b2:31:67:38:74:fd:3f:e6:49: - 5e:53:eb:88:ae:b1:45:71:0e:67:97:3c:99:4e:c7:ea:1e:02: - 67:b4:54:ef:4f:10:55:4a:70:c0:eb:41:e4:50:d4:48:5e:70: - c5:0f:79:f2:06:3d:35:ea:ce:5d:13:8e:14:65:fc:98:21:16: - 2d:5d:6d:f8:e0:6b:c7:c6:e4:8a:ca:c9:38:1f:93:27:86:28: - ef:96:e7:ad:6c:4a:9e:10:78:48:00:f4:4a:43:dc:87:1d:e3: - d3:39:53:68 + 3b:bd:d4:39:37:b3:a8:bb:34:9f:94:c2:a0:b6:be:89:c4:1f: + 02:0c:b4:08:11:6d:8f:ff:d0:92:2a:a0:91:d9:f9:b0:a8:22: + d1:cf:7a:f3:6b:a9:b3:ac:1c:21:47:61:09:07:5c:a1:c1:4f: + 5f:14:df:ab:9b:1d:10:bf:7f:b5:20:70:51:f9:4a:6d:ae:bb: + a4:14:86:36:b8:29:1d:28:36:9c:86:45:17:0b:b1:8b:4f:1d: + 10:f9:e1:12:1e:61:f0:88:1f:b2:2e:f8:e9:d7:2f:b7:59:98: + ec:50:96:49:11:4d:3d:30:1b:50:82:41:dd:96:11:eb:f9:4d: + 1e:af:52:9a:3c:59:65:ed:b6:db:dd:98:84:9a:f6:75:ab:a1: + ab:69:a3:6d:b4:db:f3:55:05:29:fa:91:d6:bc:60:8a:9e:8b: + 38:e2:18:18:a6:b3:9f:cc:4e:d8:26:a6:7b:29:d6:52:4d:84: + 33:6a:71:b1:35:c2:6e:cd:05:44:3b:67:bc:1a:55:86:ba:b3: + 3b:80:21:76:ed:93:ce:e3:3d:c4:28:9e:a5:4d:f4:f2:17:9a: + e8:be:e5:2d:ae:3a:49:54:0f:8d:fd:e2:65:9c:f5:ea:14:1e: + 9f:2a:fd:8d:59:7b:bf:51:72:a2:0c:85:0c:b7:e6:4f:e0:f5: + f9:06:94:4b -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgEwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCXN1cGVyVXNlcjCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1DfZhA+bBbvK7bwAutJpCW -4GI47WixcEY73kT5FFGGEOvKkOeI6PmRheDdtbQUuXjjhtVUbWjsFJK0+CJbBT3t -MSVlCAWEyuYMIRJYMscaYKNP0kqeKBl8RYQAjInc3orlT4iRzKTxgUVMfcL/4sGJ -xhJzleI2vduui1poapBR3iuIX6pn9KjjY9y+GYLMnX/mjfuCviIBPVYTO1sEtOjF -GOYuDfq6So3oxlqhUZpKYtev3bT84tXNrplsXGFWC9cMGndc9TpqVLWeM6ypdSia -dq/QelcAG5ETMf1CiCFHBRABL1m7xzrZ4VhMG2xxtpjv3QOCWKMy3JChtqYe4QsC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCWwiMtRtA9Iw6rPbYeMZYA664XrG7A1BqNDzZjJwJJTiTP04CIOk/Q -8eUc3y2Kq66NSHeg0NzVgByhPQ1JZL/LOYTJ813gLbqg8qwDhUShl2sL3u2nSRlG -shhJIWJDUjZvR2wha15BhShxbCInNXaC7ayt1/qdTH1vRH4G3YoRMgzZ0PZjKkCu -DVrfnteRitstlfMZ8I8eNOOyMWc4dP0/5kleU+uIrrFFcQ5nlzyZTsfqHgJntFTv -TxBVSnDA60HkUNRIXnDFD3nyBj016s5dE44UZfyYIRYtXW344GvHxuSKysk4H5Mn -hijvluetbEqeEHhIAPRKQ9yHHePTOVNo +MIIDCTCCAfGgAwIBAgIJANegMncDqPw4MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1osXBGO95E ++RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgFhMrmDCES +WDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5XiNr3brota +aGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36ukqN6MZa +oVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpXABuREzH9 +QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAO73U +OTezqLs0n5TCoLa+icQfAgy0CBFtj//Qkiqgkdn5sKgi0c9682ups6wcIUdhCQdc +ocFPXxTfq5sdEL9/tSBwUflKba67pBSGNrgpHSg2nIZFFwuxi08dEPnhEh5h8Igf +si746dcvt1mY7FCWSRFNPTAbUIJB3ZYR6/lNHq9SmjxZZe22292YhJr2dauhq2mj +bbTb81UFKfqR1rxgip6LOOIYGKazn8xO2CameynWUk2EM2pxsTXCbs0FRDtnvBpV +hrqzO4Ahdu2TzuM9xCiepU308hea6L7lLa46SVQPjf3iZZz16hQenyr9jVl7v1Fy +ogyFDLfmT+D1+QaUSw== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem index c09434c85d20a..333e1b9b80ac1 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:02 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114489 (0xd7a0327703a8fc39) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = localhost + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:af:bf:b7:2d:98:ad:9d:f6:da:a3:13:d4:62:0f: 98:be:1c:a2:89:22:ba:6f:d5:fd:1f:67:e3:91:03: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 88:89:d7:52:b3:61:49:73:7d:ee:aa:6f:47:11:cd:52:f1:ef: - 9a:63:5f:43:a9:4f:66:c8:36:dd:44:24:ba:4f:c3:6c:94:90: - 85:5e:29:fb:65:cf:03:3b:37:16:5e:88:07:70:97:54:93:f0: - f3:09:d7:65:60:09:00:fd:7f:dd:6a:ab:25:3a:30:c4:89:34: - 43:82:f6:f5:f4:2d:39:3d:21:90:c4:00:27:c5:6a:23:41:20: - c6:42:35:56:91:17:fa:31:90:09:6a:4c:e4:a7:53:ae:61:b6: - d3:5b:82:71:08:d0:0b:af:34:0f:9b:bd:bc:8c:1c:31:43:43: - 97:82:9a:ac:2a:53:ca:11:ce:6f:64:ac:86:c1:f0:62:14:aa: - c3:dd:15:5b:1c:02:6f:bb:40:87:17:b7:e5:9d:93:9a:51:c9: - 1e:7a:8c:d1:22:75:44:f1:9d:90:4b:3e:1f:6c:ab:6f:e3:be: - cd:c7:15:9d:04:84:4a:1b:a7:ac:64:5d:d7:3e:23:98:b9:49: - dd:85:dd:80:4c:46:08:9b:f5:df:eb:19:c8:57:70:ac:43:f9: - d6:9c:1b:1b:2a:94:cf:c1:35:56:a2:f4:b1:00:5d:9e:1e:36: - 54:72:ab:aa:ef:49:b2:f0:dc:cf:5b:22:51:bf:e4:c9:57:dc: - d0:48:0d:f2 + 84:4c:f6:9f:40:bd:44:a2:52:f8:62:62:98:a3:8c:78:17:fb: + da:71:cb:ca:21:34:b5:98:22:88:31:12:56:7a:d3:f2:88:2c: + fe:4c:ea:b5:bc:40:f9:5b:cf:06:6d:bd:58:3f:d9:69:99:54: + e6:5d:3a:6a:4f:92:3b:02:0f:15:01:99:d4:01:86:8f:09:c3: + a8:b6:f0:c1:21:55:9b:25:c1:2a:73:ee:b5:9b:2c:97:e6:9e: + a5:f7:b6:52:4a:6a:51:13:06:1b:5a:47:13:2c:ac:26:44:05: + 44:be:47:03:33:3d:15:fc:17:91:f2:2a:44:7d:cc:b1:3c:ac: + 31:ee:48:e9:3d:1d:a6:5f:d3:60:6f:d8:e5:1c:e4:bc:0d:a4: + dc:8d:4b:4f:e2:e4:87:fe:56:00:67:86:2b:61:c1:e0:da:eb: + 57:56:d1:43:24:15:4c:8a:4a:ac:31:74:ab:46:3e:a6:6e:f6: + 3a:09:c8:bb:ae:1c:ff:17:c1:2a:33:2c:e2:0f:d4:25:71:bc: + 9b:51:28:2f:c4:bb:44:67:86:81:2d:21:f1:22:54:e8:45:09: + 39:7b:e2:19:f6:85:0d:76:c8:2a:ca:a6:e1:d2:c5:f4:49:fe: + 02:f3:8d:cc:e6:23:19:4b:b8:f4:e4:76:91:a9:6d:5a:30:0e: + 7f:00:cb:93 -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCWxvY2FsaG9zdDCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+/ty2YrZ322qMT1GIPmL4c -ookium/V/R9n45EDmICBDu3Y9nB/LDZoPVPqWDqm1YlmS70eV3ETbUsR5UCldoQk -kkBYgJbJHyzEVeujeXNwXDeaie0vumvjgnxpSgJUi4FePL9MisvqLF6D57cQCF+C -WKOJ0dqSuioo7jAoP1uuEHGWx+ESxbAarURvRDoRSpo8D40GgHs07z9s9F7FRFQe -yN3HgIWA2WjmxlMDd+H+GGEHdwVM7Vm8XUE4au9dobJgmNRIKJUCig79z3sb0hHM -EAxQc9fMOGyD3XkmqpDIm4SGvFnpYmn0mBvEgHh+oBqBndLhZt3EzPxjBKzspzUC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCIiddSs2FJc33uqm9HEc1S8e+aY19DqU9myDbdRCS6T8NslJCFXin7 -Zc8DOzcWXogHcJdUk/DzCddlYAkA/X/daqslOjDEiTRDgvb19C05PSGQxAAnxWoj -QSDGQjVWkRf6MZAJakzkp1OuYbbTW4JxCNALrzQPm728jBwxQ0OXgpqsKlPKEc5v -ZKyGwfBiFKrD3RVbHAJvu0CHF7flnZOaUckeeozRInVE8Z2QSz4fbKtv477NxxWd -BIRKG6esZF3XPiOYuUndhd2ATEYIm/Xf6xnIV3CsQ/nWnBsbKpTPwTVWovSxAF2e -HjZUcquq70my8NzPWyJRv+TJV9zQSA3y +MIIDCTCCAfGgAwIBAgIJANegMncDqPw5MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv1f0fZ+OR +A5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICWyR8sxFXr +o3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHakroqKO4w +KD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CFgNlo5sZT +A3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPXzDhsg915 +JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAhEz2 +n0C9RKJS+GJimKOMeBf72nHLyiE0tZgiiDESVnrT8ogs/kzqtbxA+VvPBm29WD/Z +aZlU5l06ak+SOwIPFQGZ1AGGjwnDqLbwwSFVmyXBKnPutZssl+aepfe2UkpqURMG +G1pHEyysJkQFRL5HAzM9FfwXkfIqRH3MsTysMe5I6T0dpl/TYG/Y5RzkvA2k3I1L +T+Lkh/5WAGeGK2HB4NrrV1bRQyQVTIpKrDF0q0Y+pm72OgnIu64c/xfBKjMs4g/U +JXG8m1EoL8S7RGeGgS0h8SJU6EUJOXviGfaFDXbIKsqm4dLF9En+AvONzOYjGUu4 +9OR2kaltWjAOfwDLkw== -----END CERTIFICATE----- From d4be954dedcc7537b3d65b9a1d7b5662e6062fdf Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 22 Feb 2023 15:07:18 -0600 Subject: [PATCH 006/404] [fix][broker] Allow proxy to pass same role for authRole and originalRole (#19557) ### Motivation I broke the Pulsar Proxy with #19455 because that PR requires that when `X-Original-Principal` is supplied, the auth role must be a proxy role. This is not always the case for proxied admin requests. This PR seeks to fix that incorrect assumption by changing the way verification is done for the roles. Specifically, when the two roles are the same and they are not a proxy role, we will consider it a valid combination. Note that there is no inefficiency in this solution because When the `authenticatedPrincipal` is not a proxy role, that is the only role that is authenticated. Note also that we do not let the binary protocol authenticate this way, and that is consistent with the way the pulsar proxy forwards authentication data. Currently, we do the following when authentication is enabled in the proxy: 1. Authenticate the client's http request and put the resulting role in the `X-Original-Principal` header for the call to the broker. https://github.com/apache/pulsar/blob/38555851359f9cfc172650c387a58c5a03809e97/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java#L370-L373 2. Copy the `Authorization` header into the broker's http request: https://github.com/apache/pulsar/blob/38555851359f9cfc172650c387a58c5a03809e97/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java#L232-L236 3. Configure the proxy's http client to use client TLS authentication (when configured): https://github.com/apache/pulsar/blob/38555851359f9cfc172650c387a58c5a03809e97/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java#L269-L277 The problem with #19455 is that it assumes the proxy supplies its own authentication data. However, that only happens when using TLS authentication. Otherwise, the proxy forwards the client's authentication data in the `Authorization` header. As such, calls will fail because the `X-Original-Principal` header supplied without using a proxy role. ### Modifications * Consider the `authenticatedPrincipal` and the `originalPrincipal` a valid pair when they are equal and are not a `proxyRole` for http requests. ### Alternative Solutions I initially proposed that we only add the `X-Original-Principal` when we are using the proxy's authentication (see the first commit). I decided this solution is not ideal because it doesn't solve the problem, it doesn't make the brokers backwards compatible, and there isn't actually any inefficiency in passing the role as a header. ### Verifying this change When cherry-picking #19455 to branch-2.9, I discovered that `PackagesOpsWithAuthTest#testPackagesOps` was consistently failing because of the way the proxy supplies authentication data when proxying http requests. That test was removed by https://github.com/apache/pulsar/pull/12771, which explains why I didn't catch the error sooner. This PR includes a test that fails without this change. Note that the primary issue must be that we didn't have any tests doing authentication forwarding through the proxy. Now we will have both relevant tests where the proxy is and is not authenticating. ### Does this pull request potentially affect one of the following parts: This is not a breaking change. ### Documentation - [x] `doc-required` ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/31 --- .../authorization/AuthorizationService.java | 26 +++- .../pulsar/broker/service/ServerCnx.java | 2 +- .../pulsar/broker/auth/AuthorizationTest.java | 30 ++-- .../pulsar/broker/service/ServerCnxTest.java | 2 + .../server/ProxyWithAuthorizationTest.java | 143 ++++++++++-------- .../server/ProxyWithJwtAuthorizationTest.java | 14 +- 6 files changed, 129 insertions(+), 88 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 39f401b493f17..6fff04b33b618 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -292,23 +292,36 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return provider.allowSinkOpsAsync(namespaceName, role, authenticationData); } + /** + * Whether the authenticatedPrincipal and the originalPrincipal form a valid pair. This method assumes that + * authenticatedPrincipal and originalPrincipal can be equal, as long as they are not a proxy role. This use + * case is relvant for the admin server because of the way the proxy handles authentication. The binary protocol + * should not use this method. + * @return true when roles are a valid combination and false when roles are an invalid combination + */ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, String originalPrincipal, AuthenticationDataSource authDataSource) { SocketAddress remoteAddress = authDataSource != null ? authDataSource.getPeerAddress() : null; - return isValidOriginalPrincipal(authenticatedPrincipal, originalPrincipal, remoteAddress); + return isValidOriginalPrincipal(authenticatedPrincipal, originalPrincipal, remoteAddress, true); } /** * Validates that the authenticatedPrincipal and the originalPrincipal are a valid combination. - * Valid combinations fulfill the following rule: the authenticatedPrincipal is in - * {@link ServiceConfiguration#getProxyRoles()}, if, and only if, the originalPrincipal is set to a role - * that is not also in {@link ServiceConfiguration#getProxyRoles()}. + * Valid combinations fulfill one of the following two rules: + *

+ * 1. The authenticatedPrincipal is in {@link ServiceConfiguration#getProxyRoles()}, if, and only if, + * the originalPrincipal is set to a role that is not also in {@link ServiceConfiguration#getProxyRoles()}. + *

+ * 2. The authenticatedPrincipal and the originalPrincipal are the same, but are not a proxyRole, when + * allowNonProxyPrincipalsToBeEqual is true. + * * @return true when roles are a valid combination and false when roles are an invalid combination */ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, String originalPrincipal, - SocketAddress remoteAddress) { + SocketAddress remoteAddress, + boolean allowNonProxyPrincipalsToBeEqual) { String errorMsg = null; if (conf.getProxyRoles().contains(authenticatedPrincipal)) { if (StringUtils.isBlank(originalPrincipal)) { @@ -316,7 +329,8 @@ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, } else if (conf.getProxyRoles().contains(originalPrincipal)) { errorMsg = "originalPrincipal cannot be a proxy role."; } - } else if (StringUtils.isNotBlank(originalPrincipal)) { + } else if (StringUtils.isNotBlank(originalPrincipal) + && !(allowNonProxyPrincipalsToBeEqual && originalPrincipal.equals(authenticatedPrincipal))) { errorMsg = "cannot specify originalPrincipal when connecting without valid proxy role."; } if (errorMsg != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d6a6dda402eca..adee29c5a0105 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -673,7 +673,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { if (service.isAuthenticationEnabled()) { if (service.isAuthorizationEnabled()) { if (!service.getAuthorizationService() - .isValidOriginalPrincipal(authRole, originalPrincipal, remoteAddress)) { + .isValidOriginalPrincipal(authRole, originalPrincipal, remoteAddress, false)) { state = State.Failed; service.getPulsarStats().recordConnectionCreateFail(); final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError, "Invalid roles."); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index c578d9ec94162..58cf4ee418ea4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -242,27 +242,31 @@ public void testOriginalRoleValidation() throws Exception { AuthorizationService auth = new AuthorizationService(conf, Mockito.mock(PulsarResources.class)); // Original principal should be supplied when authenticatedPrincipal is proxy role - assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (SocketAddress) null, false)); // Non proxy role should not supply originalPrincipal - assertTrue(auth.isValidOriginalPrincipal("client", "", (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal("client", null, (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("client", "", (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal("client", null, (SocketAddress) null, false)); + + // Edge cases that differ because binary protocol and http protocol have different expectations + assertTrue(auth.isValidOriginalPrincipal("client", "client", (SocketAddress) null, true)); + assertFalse(auth.isValidOriginalPrincipal("client", "client", (SocketAddress) null, false)); // Only likely in cases when authentication is disabled, but we still define these to be valid. - assertTrue(auth.isValidOriginalPrincipal(null, null, (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal(null, "", (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal("", null, (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal("", "", (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal(null, null, (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal(null, "", (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal("", null, (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal("", "", (SocketAddress) null, false)); // Proxy role must supply an original principal - assertFalse(auth.isValidOriginalPrincipal("proxy", "", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal("proxy", null, (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("proxy", "", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal("proxy", null, (SocketAddress) null, false)); // OriginalPrincipal cannot be proxy role - assertFalse(auth.isValidOriginalPrincipal("proxy", "proxy", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal("client", "proxy", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal("", "proxy", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal(null, "proxy", (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("proxy", "proxy", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal("client", "proxy", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal("", "proxy", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal(null, "proxy", (SocketAddress) null, false)); // Must gracefully handle a missing AuthenticationDataSource assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (AuthenticationDataSource) null)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index ab13b8aa3c7e1..a29d6dac72023 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -613,6 +613,8 @@ public void testConnectCommandWithInvalidRoleCombinations() throws Exception { verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.proxy"); // Invalid combinations where the original principal is set to a non-proxy role verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client1", "pass.client"); + verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.client"); + verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.client1"); } private void verifyAuthRoleAndOriginalPrincipalBehavior(String authMethodName, String authData, diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index de9bb087d3da3..31757cc036720 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -84,6 +84,7 @@ public class ProxyWithAuthorizationTest extends ProducerConsumerBase { private final String TLS_SUPERUSER_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; private ProxyService proxyService; + private WebServer webServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); @DataProvider(name = "hostnameVerification") @@ -175,6 +176,7 @@ protected void doInitConf() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("superUser"); + superUserRoles.add("Proxy"); conf.setSuperUserRoles(superUserRoles); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); @@ -202,12 +204,11 @@ protected void setup() throws Exception { proxyConfig.setForwardAuthorizationCredentials(true); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); + proxyConfig.setBrokerWebServiceURLTLS(pulsar.getWebServiceAddressTls()); proxyConfig.setAdvertisedAddress(null); - proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); - proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); @@ -225,9 +226,10 @@ protected void setup() throws Exception { properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); proxyConfig.setProperties(properties); - proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + AuthenticationService authService = + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authService)); + webServer = new WebServer(proxyConfig, authService); } @AfterMethod(alwaysRun = true) @@ -235,10 +237,13 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + webServer.stop(); } private void startProxy() throws Exception { proxyService.start(); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null); + webServer.start(); } /** @@ -260,23 +265,15 @@ public void testProxyAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); startProxy(); - createAdminClient(); + // Skip hostname verification because the certs intentionally do not have a hostname + createProxyAdminClient(false); // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), PulsarClient.builder()); String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + initializeCluster(admin, namespaceName); Consumer consumer = proxyClient.newConsumer() .topic("persistent://my-tenant/my-ns/my-topic1") @@ -313,7 +310,8 @@ public void testTlsHostVerificationProxyToClient(boolean hostnameVerificationEna log.info("-- Starting {} test --", methodName); startProxy(); - createAdminClient(); + // Testing client to proxy hostname verification, so use the dataProvider's value here + createProxyAdminClient(hostnameVerificationEnabled); // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), @@ -321,17 +319,21 @@ public void testTlsHostVerificationProxyToClient(boolean hostnameVerificationEna String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder() - .serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + try { + initializeCluster(admin, namespaceName); + if (hostnameVerificationEnabled) { + Assert.fail("Connection should be failed due to hostnameVerification enabled"); + } + } catch (PulsarAdminException e) { + if (!hostnameVerificationEnabled) { + Assert.fail("Cluster should initialize because hostnameverification is disabled"); + } + admin.close(); + // Need new client because the admin client to proxy is failing due to hostname verification, and we still + // want to test the binary protocol client fails to connect as well + createProxyAdminClient(false); + initializeCluster(admin, namespaceName); + } try { proxyClient.newConsumer().topic("persistent://my-tenant/my-ns/my-topic1") @@ -366,7 +368,8 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna proxyConfig.setTlsHostnameVerificationEnabled(hostnameVerificationEnabled); startProxy(); - createAdminClient(); + // This test skips hostname verification for client to proxy in order to test proxy to broker + createProxyAdminClient(false); // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), @@ -374,16 +377,22 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + try { + initializeCluster(admin, namespaceName); + if (hostnameVerificationEnabled) { + Assert.fail("Connection should be failed due to hostnameVerification enabled for proxy to broker"); + } + } catch (PulsarAdminException.ServerSideErrorException e) { + if (!hostnameVerificationEnabled) { + Assert.fail("Cluster should initialize because hostnameverification is disabled for proxy to broker"); + } + Assert.assertEquals(e.getStatusCode(), 502, "Should get bad gateway"); + admin.close(); + // Need to use broker's admin client because the proxy to broker is failing, and we still want to test + // the binary protocol client fails to connect as well + createBrokerAdminClient(); + initializeCluster(admin, namespaceName); + } try { proxyClient.newConsumer().topic("persistent://my-tenant/my-ns/my-topic1") @@ -411,19 +420,9 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc throws Exception { log.info("-- Starting {} test --", methodName); String namespaceName = "my-tenant/my-ns"; - createAdminClient(); - - admin.clusters().createCluster("proxy-authorization", ClusterData.builder() - .serviceUrlTls(brokerUrlTls.toString()).build()); + createBrokerAdminClient(); - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + initializeCluster(admin, namespaceName); ProxyConfiguration proxyConfig = new ProxyConfiguration(); proxyConfig.setAuthenticationEnabled(true); @@ -510,7 +509,8 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception log.info("-- Starting {} test --", methodName); startProxy(); - createAdminClient(); + // Skip hostname verification because the certs intentionally do not have a hostname + createProxyAdminClient(false); @Cleanup PulsarClient proxyClient = PulsarClient.builder() @@ -525,17 +525,7 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", - ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + initializeCluster(admin, namespaceName); Consumer consumer = proxyClient.newConsumer() .topic("persistent://my-tenant/my-ns/my-topic1") @@ -567,7 +557,32 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception log.info("-- Exiting {} test --", methodName); } - private void createAdminClient() throws Exception { + private void initializeCluster(PulsarAdmin adminClient, String namespaceName) throws Exception { + adminClient.clusters().createCluster("proxy-authorization", ClusterData.builder() + .serviceUrlTls(brokerUrlTls.toString()).build()); + + adminClient.tenants().createTenant("my-tenant", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); + adminClient.namespaces().createNamespace(namespaceName); + + adminClient.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", + Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + adminClient.namespaces().grantPermissionOnNamespace(namespaceName, "Client", + Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + } + + private void createProxyAdminClient(boolean enableTlsHostnameVerification) throws Exception { + Map authParams = Maps.newHashMap(); + authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); + authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); + + admin = spy(PulsarAdmin.builder().serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .enableTlsHostnameVerification(enableTlsHostnameVerification) + .authentication(AuthenticationTls.class.getName(), authParams).build()); + } + + private void createBrokerAdminClient() throws Exception { Map authParams = Maps.newHashMap(); authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java index f42cbe4c30e87..e912006faa022 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java @@ -72,6 +72,7 @@ public class ProxyWithJwtAuthorizationTest extends ProducerConsumerBase { private final String CLIENT_TOKEN = Jwts.builder().setSubject(CLIENT_ROLE).signWith(SECRET_KEY).compact(); private ProxyService proxyService; + private WebServer webServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); @BeforeMethod @@ -105,6 +106,7 @@ protected void setup() throws Exception { proxyConfig.setAuthorizationEnabled(false); proxyConfig.getProperties().setProperty("tokenSecretKey", "data:;base64," + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setBrokerWebServiceURL(pulsar.getWebServiceAddress()); proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); @@ -115,9 +117,10 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientAuthenticationParameters(PROXY_TOKEN); proxyConfig.setAuthenticationProviders(providers); - proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + AuthenticationService authService = + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authService)); + webServer = new WebServer(proxyConfig, authService); } @AfterMethod(alwaysRun = true) @@ -125,10 +128,13 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + webServer.stop(); } private void startProxy() throws Exception { proxyService.start(); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null); + webServer.start(); } /** @@ -435,7 +441,7 @@ void testGetMetrics() throws Exception { } private void createAdminClient() throws Exception { - admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) + admin = spy(PulsarAdmin.builder().serviceHttpUrl(webServer.getServiceUri().toString()) .authentication(AuthenticationFactory.token(ADMIN_TOKEN)).build()); } From e6bc4999e22a763c9097494970ed6d9796278a23 Mon Sep 17 00:00:00 2001 From: AloysZhang Date: Thu, 23 Feb 2023 13:18:10 +0800 Subject: [PATCH 007/404] [fix][txn]fix receive duplicated messages due to pendingAcks in PendingAckHandle (#19581) Co-authored-by: mayozhang --- .../mledger/util/PositionAckSetUtil.java | 7 ++ .../service/AbstractBaseDispatcher.java | 13 ++++ .../client/impl/TransactionEndToEndTest.java | 78 +++++++++++++++++++ .../BitSetRecyclableRecyclableTest.java | 17 ++++ 4 files changed, 115 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java index 8173b30c4fea9..1c607582076a8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java @@ -59,6 +59,13 @@ public static long[] andAckSet(long[] firstAckSet, long[] secondAckSet) { return ackSet; } + public static boolean isAckSetEmpty(long[] ackSet) { + BitSetRecyclable bitSet = BitSetRecyclable.create().resetWords(ackSet); + boolean isEmpty = bitSet.isEmpty(); + bitSet.recycle(); + return isEmpty; + } + //This method is compare two position which position is bigger than another one. //When the ledgerId and entryId in this position is same to another one and two position all have ack set, it will //compare the ack set next bit index is bigger than another one. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index ef2fd80302a98..8f6caa7a20801 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.andAckSet; +import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.isAckSetEmpty; import io.netty.buffer.ByteBuf; import io.prometheus.client.Gauge; import java.util.ArrayList; @@ -239,6 +240,18 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i // if actSet is null, use pendingAck ackSet ackSet = positionInPendingAck.getAckSet(); } + // if the result of pendingAckSet(in pendingAckHandle) AND the ackSet(in cursor) is empty + // filter this entry + if (isAckSetEmpty(ackSet)) { + entries.set(i, null); + entry.release(); + continue; + } + } else { + // filter non-batch message in pendingAck state + entries.set(i, null); + entry.release(); + continue; } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 527b8532e0452..83feaa3ac1158 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -176,6 +176,84 @@ private void testIndividualAckAbortFilterAckSetInPendingAckState() throws Except assertNull(consumer.receive(2, TimeUnit.SECONDS)); } + + @Test(dataProvider="enableBatch") + private void testFilterMsgsInPendingAckStateWhenConsumerDisconnect(boolean enableBatch) throws Exception { + final String topicName = NAMESPACE1 + "/testFilterMsgsInPendingAckStateWhenConsumerDisconnect-" + enableBatch; + final int count = 10; + + @Cleanup + Producer producer = null; + if (enableBatch) { + producer = pulsarClient + .newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .batchingMaxMessages(count).create(); + } else { + producer = pulsarClient + .newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(false).create(); + } + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName("test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + for (int i = 0; i < count; i++) { + producer.sendAsync(i); + } + + Transaction txn1 = getTxn(); + + Transaction txn2 = getTxn(); + + + // txn1 ack half of messages and don't end the txn1 + for (int i = 0; i < count / 2; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), txn1).get(); + } + + // txn2 ack the rest half of messages and commit tnx2 + for (int i = count / 2; i < count; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), txn2).get(); + } + // commit txn2 + txn2.commit().get(); + + // close and re-create consumer + consumer.close(); + consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName("test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Message message = consumer.receive(3, TimeUnit.SECONDS); + Assert.assertNull(message); + + // abort txn1 + txn1.abort().get(); + // after txn1 aborted, consumer will receive messages txn1 contains + int receiveCounter = 0; + while((message = consumer.receive(3, TimeUnit.SECONDS)) != null) { + Assert.assertEquals(message.getValue().intValue(), receiveCounter); + receiveCounter ++; + } + Assert.assertEquals(receiveCounter, count / 2); + } + @Test(dataProvider="enableBatch") private void produceCommitTest(boolean enableBatch) throws Exception { @Cleanup diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java index 8374f2db8961a..8061f853d66c1 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java @@ -45,4 +45,21 @@ public void testResetWords() { Assert.assertTrue(bitset1.get(128)); Assert.assertFalse(bitset1.get(256)); } + + @Test + public void testBitSetEmpty() { + BitSetRecyclable bitSet = BitSetRecyclable.create(); + bitSet.set(0, 5); + bitSet.clear(1); + bitSet.clear(2); + bitSet.clear(3); + long[] array = bitSet.toLongArray(); + Assert.assertFalse(bitSet.isEmpty()); + Assert.assertFalse(BitSetRecyclable.create().resetWords(array).isEmpty()); + bitSet.clear(0); + bitSet.clear(4); + Assert.assertTrue(bitSet.isEmpty()); + long[] array1 = bitSet.toLongArray(); + Assert.assertTrue(BitSetRecyclable.create().resetWords(array1).isEmpty()); + } } From e2863391e7f6f9b6c5060f0f78378493f8df37f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 23 Feb 2023 10:12:10 +0100 Subject: [PATCH 008/404] [fix][client] Broker address resolution wrong if connect through a multi-dns names proxy (#19597) --- .../client/impl/ConnectionPoolTest.java | 89 ++++++++++++++++++- .../pulsar/client/impl/ConnectionPool.java | 24 +++-- .../client/impl/PulsarChannelInitializer.java | 8 +- 3 files changed, 106 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index e8816894513d2..fb564bd5083c1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -20,12 +20,17 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import io.netty.channel.EventLoopGroup; +import io.netty.resolver.AbstractAddressResolver; import io.netty.util.concurrent.DefaultThreadFactory; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; import java.util.stream.IntStream; +import io.netty.util.concurrent.Promise; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.util.netty.EventLoopUtil; @@ -66,7 +71,7 @@ public void testSingleIpAddress() throws Exception { List result = new ArrayList<>(); result.add(new InetSocketAddress("127.0.0.1", brokerPort)); Mockito.when(pool.resolveName(InetSocketAddress.createUnresolved("non-existing-dns-name", - brokerPort))) + brokerPort))) .thenReturn(CompletableFuture.completedFuture(result)); client.newProducer().topic("persistent://sample/standalone/ns/my-topic").create(); @@ -107,7 +112,7 @@ public void testNoConnectionPool() throws Exception { ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = - InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); + InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); IntStream.range(1, 5).forEach(i -> { pool.getConnection(brokerAddress).thenAccept(cnx -> { Assert.assertTrue(cnx.channel().isActive()); @@ -119,6 +124,7 @@ public void testNoConnectionPool() throws Exception { pool.closeAllConnections(); pool.close(); + eventLoop.shutdownGracefully(); } @Test @@ -129,7 +135,7 @@ public void testEnableConnectionPool() throws Exception { ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = - InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); + InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); IntStream.range(1, 10).forEach(i -> { pool.getConnection(brokerAddress).thenAccept(cnx -> { Assert.assertTrue(cnx.channel().isActive()); @@ -141,5 +147,82 @@ public void testEnableConnectionPool() throws Exception { pool.closeAllConnections(); pool.close(); + eventLoop.shutdownGracefully(); + } + + + @Test + public void testSetProxyToTargetBrokerAddress() throws Exception { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setConnectionsPerBroker(5); + + + EventLoopGroup eventLoop = + EventLoopUtil.newEventLoopGroup(8, false, + new DefaultThreadFactory("test")); + + final AbstractAddressResolver resolver = new AbstractAddressResolver(eventLoop.next()) { + @Override + protected boolean doIsResolved(SocketAddress socketAddress) { + return !((InetSocketAddress) socketAddress).isUnresolved(); + } + + @Override + protected void doResolve(SocketAddress socketAddress, Promise promise) throws Exception { + promise.setFailure(new IllegalStateException()); + throw new IllegalStateException(); + } + + @Override + protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws Exception { + final InetSocketAddress socketAddress1 = (InetSocketAddress) socketAddress; + final boolean isProxy = socketAddress1.getHostName().equals("proxy"); + final boolean isBroker = socketAddress1.getHostName().equals("broker"); + if (!isProxy && !isBroker) { + promise.setFailure(new IllegalStateException()); + throw new IllegalStateException(); + } + List result = new ArrayList<>(); + if (isProxy) { + result.add(new InetSocketAddress("localhost", brokerPort)); + result.add(InetSocketAddress.createUnresolved("proxy", brokerPort)); + } else { + result.add(new InetSocketAddress("127.0.0.1", brokerPort)); + result.add(InetSocketAddress.createUnresolved("broker", brokerPort)); + } + promise.setSuccess(result); + } + }; + + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop, + (Supplier) () -> new ClientCnx(conf, eventLoop), Optional.of(resolver)); + + + ClientCnx cnx = pool.getConnection( + InetSocketAddress.createUnresolved("proxy", 9999), + InetSocketAddress.createUnresolved("proxy", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "proxy"); + Assert.assertNull(cnx.proxyToTargetBrokerAddress); + cnx.close(); + + cnx = pool.getConnection( + InetSocketAddress.createUnresolved("broker", 9999), + InetSocketAddress.createUnresolved("proxy", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "proxy"); + Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker:9999"); + cnx.close(); + + + cnx = pool.getConnection( + InetSocketAddress.createUnresolved("broker", 9999), + InetSocketAddress.createUnresolved("broker", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "broker"); + Assert.assertNull(cnx.proxyToTargetBrokerAddress); + cnx.close(); + + + pool.closeAllConnections(); + pool.close(); + eventLoop.shutdownGracefully(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 2e105b5328467..3a9a2b9b7ab94 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -305,8 +305,12 @@ private CompletableFuture createConnection(InetSocketAddress logicalAdd resolvedAddress = resolveName(unresolvedPhysicalAddress); } return resolvedAddress.thenCompose( - inetAddresses -> connectToResolvedAddresses(logicalAddress, inetAddresses.iterator(), - isSniProxy ? unresolvedPhysicalAddress : null)); + inetAddresses -> connectToResolvedAddresses( + logicalAddress, + unresolvedPhysicalAddress, + inetAddresses.iterator(), + isSniProxy ? unresolvedPhysicalAddress : null) + ); } catch (URISyntaxException e) { log.error("Invalid Proxy url {}", clientConfig.getProxyServiceUrl(), e); return FutureUtil @@ -319,17 +323,19 @@ private CompletableFuture createConnection(InetSocketAddress logicalAdd * address is working. */ private CompletableFuture connectToResolvedAddresses(InetSocketAddress logicalAddress, + InetSocketAddress unresolvedPhysicalAddress, Iterator resolvedPhysicalAddress, InetSocketAddress sniHost) { CompletableFuture future = new CompletableFuture<>(); // Successfully connected to server - connectToAddress(logicalAddress, resolvedPhysicalAddress.next(), sniHost) + connectToAddress(logicalAddress, resolvedPhysicalAddress.next(), unresolvedPhysicalAddress, sniHost) .thenAccept(future::complete) .exceptionally(exception -> { if (resolvedPhysicalAddress.hasNext()) { // Try next IP address - connectToResolvedAddresses(logicalAddress, resolvedPhysicalAddress, sniHost) + connectToResolvedAddresses(logicalAddress, unresolvedPhysicalAddress, + resolvedPhysicalAddress, sniHost) .thenAccept(future::complete) .exceptionally(ex -> { // This is already unwinding the recursive call @@ -362,20 +368,24 @@ CompletableFuture> resolveName(InetSocketAddress unresol * Attempt to establish a TCP connection to an already resolved single IP address. */ private CompletableFuture connectToAddress(InetSocketAddress logicalAddress, - InetSocketAddress physicalAddress, InetSocketAddress sniHost) { + InetSocketAddress physicalAddress, + InetSocketAddress unresolvedPhysicalAddress, + InetSocketAddress sniHost) { if (clientConfig.isUseTls()) { return toCompletableFuture(bootstrap.register()) .thenCompose(channel -> channelInitializerHandler .initTls(channel, sniHost != null ? sniHost : physicalAddress)) .thenCompose(channelInitializerHandler::initSocks5IfConfig) .thenCompose(ch -> - channelInitializerHandler.initializeClientCnx(ch, logicalAddress, physicalAddress)) + channelInitializerHandler.initializeClientCnx(ch, logicalAddress, + unresolvedPhysicalAddress)) .thenCompose(channel -> toCompletableFuture(channel.connect(physicalAddress))); } else { return toCompletableFuture(bootstrap.register()) .thenCompose(channelInitializerHandler::initSocks5IfConfig) .thenCompose(ch -> - channelInitializerHandler.initializeClientCnx(ch, logicalAddress, physicalAddress)) + channelInitializerHandler.initializeClientCnx(ch, logicalAddress, + unresolvedPhysicalAddress)) .thenCompose(channel -> toCompletableFuture(channel.connect(physicalAddress))); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index e01b53b8ef136..ed34f7d41c130 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -213,7 +213,7 @@ CompletableFuture initSocks5IfConfig(Channel ch) { CompletableFuture initializeClientCnx(Channel ch, InetSocketAddress logicalAddress, - InetSocketAddress resolvedPhysicalAddress) { + InetSocketAddress unresolvedPhysicalAddress) { return NettyFutureUtil.toCompletableFuture(ch.eventLoop().submit(() -> { final ClientCnx cnx = (ClientCnx) ch.pipeline().get("handler"); @@ -221,15 +221,13 @@ CompletableFuture initializeClientCnx(Channel ch, throw new IllegalStateException("Missing ClientCnx. This should not happen."); } - // Need to do our own equality because the physical address is resolved already - if (!(logicalAddress.getHostString().equalsIgnoreCase(resolvedPhysicalAddress.getHostString()) - && logicalAddress.getPort() == resolvedPhysicalAddress.getPort())) { + if (!logicalAddress.equals(unresolvedPhysicalAddress)) { // We are connecting through a proxy. We need to set the target broker in the ClientCnx object so that // it can be specified when sending the CommandConnect. cnx.setTargetBroker(logicalAddress); } - cnx.setRemoteHostName(resolvedPhysicalAddress.getHostString()); + cnx.setRemoteHostName(unresolvedPhysicalAddress.getHostString()); return ch; })); From 0bb0f6b786d115a7405867b701521cd4a49340c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 23 Feb 2023 13:54:27 +0100 Subject: [PATCH 009/404] [fix][broker] Copy command fields and fix potential thread-safety in ServerCnx (#19517) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index adee29c5a0105..242947f6a0fd6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2438,10 +2438,11 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { final TxnID txnID = new TxnID(command.getTxnidMostBits(), command.getTxnidLeastBits()); final TransactionCoordinatorID tcId = TransactionCoordinatorID.get(command.getTxnidMostBits()); final long requestId = command.getRequestId(); + final List partitionsList = command.getPartitionsList(); if (log.isDebugEnabled()) { - command.getPartitionsList().forEach(partion -> + partitionsList.forEach(partition -> log.debug("Receive add published partition to txn request {} " - + "from {} with txnId {}, topic: [{}]", requestId, remoteAddress, txnID, partion)); + + "from {} with txnId {}, topic: [{}]", requestId, remoteAddress, txnID, partition)); } if (!checkTransactionEnableAndSendError(requestId)) { @@ -2456,7 +2457,7 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { return failedFutureTxnNotOwned(txnID); } return transactionMetadataStoreService - .addProducedPartitionToTxn(txnID, command.getPartitionsList()); + .addProducedPartitionToTxn(txnID, partitionsList); }) .whenComplete((v, ex) -> { if (ex == null) { From 389792b1fc7a56647ccfc820e83ae08dfed037df Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 23 Feb 2023 07:23:00 -0800 Subject: [PATCH 010/404] [improve][broker] PIP-192 Added Deleted and Init states in ServiceUnitState (#19546) --- .../pulsar/broker/ServiceConfiguration.java | 17 + .../ExtensibleLoadManagerWrapper.java | 4 + .../extensions/channel/ServiceUnitState.java | 67 +- .../channel/ServiceUnitStateChannelImpl.java | 513 ++++++++++----- .../ServiceUnitStateCompactionStrategy.java | 93 ++- .../channel/ServiceUnitStateData.java | 20 +- .../StrategicTwoPhaseCompactor.java | 39 +- .../ExtensibleLoadManagerImplTest.java | 187 +++--- .../channel/ServiceUnitStateChannelTest.java | 615 +++++++++++++++--- ...erviceUnitStateCompactionStrategyTest.java | 125 ++-- .../channel/ServiceUnitStateDataTest.java | 15 +- .../channel/ServiceUnitStateTest.java | 82 ++- .../ServiceUnitStateCompactionTest.java | 333 +++++++--- 13 files changed, 1508 insertions(+), 602 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 106410d855e22..4f2c8e72e131d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2450,6 +2450,7 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private long namespaceBundleUnloadingTimeoutMs = 60000; + /**** --- Load Balancer Extension. --- ****/ @FieldContext( category = CATEGORY_LOAD_BALANCER, dynamic = true, @@ -2525,6 +2526,22 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private double loadBalancerBundleLoadReportPercentage = 10; + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + doc = "After this delay, the service-unit state channel tombstones any service units (e.g., bundles) " + + "in semi-terminal states. For example, after splits, parent bundles will be `deleted`, " + + "and then after this delay, the parent bundles' state will be `tombstoned` " + + "in the service-unit state channel. " + + "Pulsar does not immediately remove such semi-terminal states " + + "to avoid unnecessary system confusion, " + + "as the bundles in the `tombstoned` state might temporarily look available to reassign. " + + "Rarely, one could lower this delay in order to aggressively clean " + + "the service-unit state channel when there are a large number of bundles. " + + "minimum value = 30 secs" + + "(only used in load balancer extension logics)" + ) + private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800; + /**** --- Replication. --- ****/ @FieldContext( category = CATEGORY_REPLICATION, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 48fc4bb7ff4f0..1eabbe620e213 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -142,4 +142,8 @@ public void doNamespaceBundleSplit() { throw new UnsupportedOperationException(); } + public ExtensibleLoadManagerImpl get() { + return loadManager; + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java index 3225c0ba7bbc7..92fef8f65992a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java @@ -24,55 +24,38 @@ /** * Defines the possible states for service units. * - * The following diagram defines the valid state changes - * - * ┌───────────┐ - * ┌──────────┤ released │◄────────┐ - * │own └───────────┘ │release - * │ │ - * │ │ - * ▼ │ - * ┌────────┐ assign(transfer) ┌─────┴────┐ - * │ ├───────────────────►│ │ - * │ owned │ │ assigned │ - * │ │◄───────────────────┤ │ - * └──┬─────┤ own └──────────┘ - * │ ▲ │ ▲ - * │ │ │ │ - * │ │ └──────────────┐ │ - * │ │ │ │ - * │ │ unload │ │ assign(assignment) - * split │ │ │ │ - * │ │ │ │ - * │ │ create(child) │ │ - * │ │ │ │ - * ▼ │ │ │ - * ┌─────┴─────┐ └─────►┌───┴──────┐ - * │ │ │ │ - * │ splitting ├────────────────► │ free │ - * │ │ discard(parent)│ │ - * └───────────┘ └──────────┘ + * @see Service Unit State Channel for additional details. */ public enum ServiceUnitState { - Free, // not owned by any broker (terminal state) + Init, // initializing the state. no previous state(terminal state) + + Free, // not owned by any broker (semi-terminal state) Owned, // owned by a broker (terminal state) - Assigned, // the ownership is assigned(but the assigned broker has not been notified the ownership yet) + Assigning, // the ownership is being assigned (e.g. the new ownership is being notified to the target broker) - Released, // the source broker's ownership has been released (e.g. the topic connections are closed) + Releasing, // the source broker's ownership is being released (e.g. the topic connections are being closed) - Splitting; // the service unit(e.g. bundle) is in the process of splitting. + Splitting, // the service unit is in the process of splitting. (e.g. the metadata store is being updated) - private static Map> validTransitions = Map.of( - // (Free -> Released | Splitting) transitions are required - // when the topic is compacted in the middle of transfer or split. - Free, Set.of(Owned, Assigned, Released, Splitting), - Owned, Set.of(Assigned, Splitting, Free), - Assigned, Set.of(Owned, Released, Free), - Released, Set.of(Owned, Free), - Splitting, Set.of(Free) + Deleted; // deleted in the system (semi-terminal state) + + private static final Map> validTransitions = Map.of( + // (Init -> all states) transitions are required + // when the topic is compacted in the middle of assign, transfer or split. + Init, Set.of(Free, Owned, Assigning, Releasing, Splitting, Deleted), + Free, Set.of(Assigning, Init), + Owned, Set.of(Assigning, Splitting, Releasing), + Assigning, Set.of(Owned, Releasing), + Releasing, Set.of(Owned, Free), + Splitting, Set.of(Deleted), + Deleted, Set.of(Init) + ); + + private static final Set inFlightStates = Set.of( + Assigning, Releasing, Splitting ); public static boolean isValidTransition(ServiceUnitState from, ServiceUnitState to) { @@ -80,4 +63,8 @@ public static boolean isValidTransition(ServiceUnitState from, ServiceUnitState return transitions.contains(to); } + public static boolean isInFlightState(ServiceUnitState state) { + return inFlightStates.contains(state); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index d10138bda6805..9f205f85c5454 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -20,10 +20,13 @@ import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Closed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Constructed; @@ -35,6 +38,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Jittery; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Stable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; import com.google.common.annotations.VisibleForTesting; @@ -61,9 +65,15 @@ import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -92,37 +102,42 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { TopicDomain.persistent.value(), NamespaceName.SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); - - // TODO: define StateCompactionStrategy private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec + + private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately - private static final long MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS = 10; private static final int MAX_OUTSTANDING_PUB_MESSAGES = 500; private final PulsarService pulsar; + private final ServiceConfiguration config; private final Schema schema; private final ConcurrentOpenHashMap> getOwnerRequests; private final String lookupServiceAddress; - // TODO: define BrokerRegistry private final ConcurrentOpenHashMap> cleanupJobs; private final LeaderElectionService leaderElectionService; + private BrokerSelectionStrategy brokerSelector; + private BrokerRegistry brokerRegistry; private TableView tableview; private Producer producer; - private ScheduledFuture cleanupTasks; + private ScheduledFuture monitorTask; private SessionEvent lastMetadataSessionEvent = SessionReestablished; private long lastMetadataSessionEventTimestamp = 0; private long inFlightStateWaitingTimeInMillis; + + private long ownershipMonitorDelayTimeInSecs; + private long semiTerminalStateWaitingTimeInMillis; private long maxCleanupDelayTimeInSecs; private long minCleanupDelayTimeInSecs; // cleanup metrics - private long totalCleanupCnt = 0; - private long totalBrokerCleanupTombstoneCnt = 0; - private long totalServiceUnitCleanupTombstoneCnt = 0; + private long totalInactiveBrokerCleanupCnt = 0; + private long totalServiceUnitTombstoneCleanupCnt = 0; + + private long totalOrphanServiceUnitCleanupCnt = 0; private AtomicLong totalCleanupErrorCnt = new AtomicLong(); - private long totalCleanupScheduledCnt = 0; - private long totalCleanupIgnoredCnt = 0; - private long totalCleanupCancelledCnt = 0; + private long totalInactiveBrokerCleanupScheduledCnt = 0; + private long totalInactiveBrokerCleanupIgnoredCnt = 0; + private long totalInactiveBrokerCleanupCancelledCnt = 0; private volatile ChannelState channelState; public enum EventType { @@ -135,30 +150,18 @@ public enum EventType { @Getter @AllArgsConstructor public static class Counters { - private AtomicLong total; - private AtomicLong failure; + private final AtomicLong total; + private final AtomicLong failure; + public Counters(){ + total = new AtomicLong(); + failure = new AtomicLong(); + } } // operation metrics - final Map ownerLookUpCounters = Map.of( - Owned, new AtomicLong(), - Assigned, new AtomicLong(), - Released, new AtomicLong(), - Splitting, new AtomicLong(), - Free, new AtomicLong() - ); - final Map eventCounters = Map.of( - Assign, new Counters(new AtomicLong(), new AtomicLong()), - Split, new Counters(new AtomicLong(), new AtomicLong()), - Unload, new Counters(new AtomicLong(), new AtomicLong()) - ); - final Map handlerCounters = Map.of( - Owned, new Counters(new AtomicLong(), new AtomicLong()), - Assigned, new Counters(new AtomicLong(), new AtomicLong()), - Released, new Counters(new AtomicLong(), new AtomicLong()), - Splitting, new Counters(new AtomicLong(), new AtomicLong()), - Free, new Counters(new AtomicLong(), new AtomicLong()) - ); + final Map ownerLookUpCounters; + final Map eventCounters; + final Map handlerCounters; enum ChannelState { Closed(0), @@ -180,25 +183,61 @@ enum MetadataState { public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.pulsar = pulsar; + this.config = pulsar.getConfig(); this.lookupServiceAddress = pulsar.getLookupServiceAddress(); this.schema = Schema.JSON(ServiceUnitStateData.class); this.getOwnerRequests = ConcurrentOpenHashMap.>newBuilder().build(); this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); + this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateCleanUpDelayTimeInSeconds() + * 1000; this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; + this.ownershipMonitorDelayTimeInSecs = OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS; + if (semiTerminalStateWaitingTimeInMillis < inFlightStateWaitingTimeInMillis) { + throw new IllegalArgumentException( + "Invalid Config: loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds < " + + (MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS / 1000) + " secs"); + } this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS; this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS; this.leaderElectionService = new LeaderElectionService( pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), state -> { if (state == LeaderElectionState.Leading) { - log.debug("This broker:{} is the leader now.", lookupServiceAddress); - // TODO: schedule monitorOwnerships by brokerRegistry + log.info("This broker:{} is the leader now.", lookupServiceAddress); + this.monitorTask = this.pulsar.getLoadManagerExecutor() + .scheduleWithFixedDelay(() -> { + try { + monitorOwnerships(brokerRegistry.getAvailableBrokersAsync() + .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); + } catch (Exception e) { + log.info("Failed to monitor the ownerships. will retry..", e); + } + }, + ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS); } else { - log.debug("This broker:{} is a follower now.", lookupServiceAddress); - // TODO: cancel scheduled monitorOwnerships if any + log.info("This broker:{} is a follower now.", lookupServiceAddress); + if (monitorTask != null) { + monitorTask.cancel(false); + monitorTask = null; + log.info("This previous leader broker:{} stopped the channel clean-up monitor", + lookupServiceAddress); + } } }); + Map tmpOwnerLookUpCounters = new HashMap<>(); + Map tmpHandlerCounters = new HashMap<>(); + Map tmpEventCounters = new HashMap<>(); + for (var state : ServiceUnitState.values()) { + tmpOwnerLookUpCounters.put(state, new AtomicLong()); + tmpHandlerCounters.put(state, new Counters()); + } + for (var event : EventType.values()) { + tmpEventCounters.put(event, new Counters()); + } + ownerLookUpCounters = Map.copyOf(tmpOwnerLookUpCounters); + handlerCounters = Map.copyOf(tmpHandlerCounters); + eventCounters = Map.copyOf(tmpEventCounters); this.channelState = Constructed; } @@ -207,14 +246,22 @@ public synchronized void start() throws PulsarServerException { throw new IllegalStateException("Invalid channel state:" + channelState.name()); } + boolean debug = debug(); try { + this.brokerRegistry = getBrokerRegistry(); + this.brokerRegistry.addListener(this::handleBrokerRegistrationEvent); leaderElectionService.start(); this.channelState = LeaderElectionServiceStarted; - log.debug("Successfully started the channel leader election service."); + if (debug) { + log.info("Successfully started the channel leader election service."); + } + brokerSelector = getBrokerSelector(); if (producer != null) { producer.close(); - log.debug("Closed the channel producer."); + if (debug) { + log.info("Closed the channel producer."); + } } producer = pulsar.getClient().newProducer(schema) .enableBatching(true) @@ -223,11 +270,15 @@ public synchronized void start() throws PulsarServerException { .topic(TOPIC) .create(); - log.debug("Successfully started the channel producer."); + if (debug) { + log.info("Successfully started the channel producer."); + } if (tableview != null) { tableview.close(); - log.debug("Closed the channel tableview."); + if (debug) { + log.info("Closed the channel tableview."); + } } tableview = pulsar.getClient().newTableViewBuilder(schema) .topic(TOPIC) @@ -236,10 +287,13 @@ public synchronized void start() throws PulsarServerException { ServiceUnitStateCompactionStrategy.class.getName())) .create(); tableview.listen((key, value) -> handle(key, value)); - log.debug("Successfully started the channel tableview."); - + if (debug) { + log.info("Successfully started the channel tableview."); + } pulsar.getLocalMetadataStore().registerSessionListener(this::handleMetadataSessionEvent); - log.debug("Successfully registered the handleMetadataSessionEvent"); + if (debug) { + log.info("Successfully registered the handleMetadataSessionEvent"); + } channelState = Started; log.info("Successfully started the channel."); @@ -250,16 +304,39 @@ public synchronized void start() throws PulsarServerException { } } + @VisibleForTesting + protected BrokerRegistry getBrokerRegistry() { + return ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get()) + .get().getBrokerRegistry(); + } + + @VisibleForTesting + protected LoadManagerContext getContext() { + return ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get()) + .get().getContext(); + } + + @VisibleForTesting + protected BrokerSelectionStrategy getBrokerSelector() { + // TODO: make this selector configurable. + return new LeastResourceUsageWithWeight(); + } + public synchronized void close() throws PulsarServerException { channelState = Closed; + boolean debug = debug(); try { leaderElectionService.close(); - log.debug("Successfully closed the channel leader election service."); + if (debug) { + log.info("Successfully closed the channel leader election service."); + } if (tableview != null) { tableview.close(); tableview = null; - log.debug("Successfully closed the channel tableview."); + if (debug) { + log.info("Successfully closed the channel tableview."); + } } if (producer != null) { @@ -268,11 +345,13 @@ public synchronized void close() throws PulsarServerException { log.info("Successfully closed the channel producer."); } - // TODO: clean brokerRegistry + if (brokerRegistry != null) { + brokerRegistry = null; + } - if (cleanupTasks != null) { - cleanupTasks.cancel(true); - cleanupTasks = null; + if (monitorTask != null) { + monitorTask.cancel(true); + monitorTask = null; log.info("Successfully cancelled the cleanup tasks"); } @@ -294,7 +373,7 @@ private boolean validateChannelState(ChannelState targetState, boolean checkLowe } private boolean debug() { - return pulsar.getConfiguration().isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); } public CompletableFuture> getChannelOwnerAsync() { @@ -348,18 +427,22 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } ServiceUnitStateData data = tableview.get(serviceUnit); - ServiceUnitState state = data == null ? Free : data.state(); + ServiceUnitState state = state(data); ownerLookUpCounters.get(state).incrementAndGet(); switch (state) { case Owned, Splitting -> { return CompletableFuture.completedFuture(Optional.of(data.broker())); } - case Assigned, Released -> { - return deferGetOwnerRequest(serviceUnit).thenApply(Optional::of); + case Assigning, Releasing -> { + return deferGetOwnerRequest(serviceUnit).thenApply( + broker -> broker == null ? Optional.empty() : Optional.of(broker)); } - case Free -> { + case Init, Free -> { return CompletableFuture.completedFuture(Optional.empty()); } + case Deleted -> { + return CompletableFuture.failedFuture(new IllegalArgumentException(serviceUnit + " is deleted.")); + } default -> { String errorMsg = String.format("Failed to process service unit state data: %s when get owner.", data); log.error(errorMsg); @@ -372,7 +455,7 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str EventType eventType = Assign; eventCounters.get(eventType).getTotal().incrementAndGet(); CompletableFuture getOwnerRequest = deferGetOwnerRequest(serviceUnit); - pubAsync(serviceUnit, new ServiceUnitStateData(Assigned, broker)) + pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker)) .whenComplete((__, ex) -> { if (ex != null) { getOwnerRequests.remove(serviceUnit, getOwnerRequest); @@ -391,11 +474,11 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { String serviceUnit = unload.serviceUnit(); CompletableFuture future; if (isTransferCommand(unload)) { - ServiceUnitStateData next = new ServiceUnitStateData(Assigned, - unload.destBroker().get(), unload.sourceBroker()); - future = pubAsync(serviceUnit, next); + future = pubAsync(serviceUnit, new ServiceUnitStateData( + Assigning, unload.destBroker().get(), unload.sourceBroker())); } else { - future = tombstoneAsync(serviceUnit); + future = pubAsync(serviceUnit, new ServiceUnitStateData( + Releasing, unload.sourceBroker())); } return future.whenComplete((__, ex) -> { @@ -424,14 +507,16 @@ private void handle(String serviceUnit, ServiceUnitStateData data) { lookupServiceAddress, serviceUnit, data, totalHandledRequests); } - ServiceUnitState state = data == null ? Free : data.state(); + ServiceUnitState state = state(data); try { switch (state) { case Owned -> handleOwnEvent(serviceUnit, data); - case Assigned -> handleAssignEvent(serviceUnit, data); - case Released -> handleReleaseEvent(serviceUnit, data); + case Assigning -> handleAssignEvent(serviceUnit, data); + case Releasing -> handleReleaseEvent(serviceUnit, data); case Splitting -> handleSplitEvent(serviceUnit, data); - case Free -> handleFreeEvent(serviceUnit); + case Deleted -> handleDeleteEvent(serviceUnit, data); + case Free -> handleFreeEvent(serviceUnit, data); + case Init -> handleInitEvent(serviceUnit); default -> throw new IllegalStateException("Failed to handle channel data:" + data); } } catch (Throwable e){ @@ -453,7 +538,7 @@ private static boolean isTransferCommand(Unload data) { } private static String getLogEventTag(ServiceUnitStateData data) { - return data == null ? "Free" : + return data == null ? Init.toString() : isTransferCommand(data) ? "Transfer:" + data.state() : data.state().toString(); } @@ -466,7 +551,7 @@ private AtomicLong getHandlerFailureCounter(ServiceUnitStateData data) { } private AtomicLong getHandlerCounter(ServiceUnitStateData data, boolean total) { - var state = data == null ? Free : data.state(); + var state = state(data); var counter = total ? handlerCounters.get(state).getTotal() : handlerCounters.get(state).getFailure(); if (counter == null) { @@ -512,22 +597,30 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { } private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { - deferGetOwnerRequest(serviceUnit); if (isTargetBroker(data.broker())) { ServiceUnitStateData next = new ServiceUnitStateData( - isTransferCommand(data) ? Released : Owned, data.broker(), data.sourceBroker()); + isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker()); pubAsync(serviceUnit, next) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTargetBroker(data.sourceBroker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker()); - // TODO: when close, pass message to clients to connect to the new broker - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) - .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + if (isTransferCommand(data)) { + if (isTargetBroker(data.sourceBroker())) { + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker()); + // TODO: when close, pass message to clients to connect to the new broker + closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)) + .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + } + } else { + if (isTargetBroker(data.broker())) { + ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker()); + closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)) + .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + } } } @@ -538,16 +631,32 @@ private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { } } - private void handleFreeEvent(String serviceUnit) { - closeServiceUnit(serviceUnit) - .thenAccept(__ -> { - var request = getOwnerRequests.remove(serviceUnit); - if (request != null) { - request.completeExceptionally(new IllegalStateException("The ownership has been unloaded. " - + "No owner is found for serviceUnit: " + serviceUnit)); - } - }) - .whenComplete((__, e) -> log(e, serviceUnit, null, null)); + private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { + var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.complete(null); + } + if (isTargetBroker(data.broker())) { + log(null, serviceUnit, data, null); + } + } + + private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { + var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); + } + if (isTargetBroker(data.broker())) { + log(null, serviceUnit, data, null); + } + } + + private void handleInitEvent(String serviceUnit) { + var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.complete(null); + } + log(null, serviceUnit, null, null); } private CompletableFuture pubAsync(String serviceUnit, ServiceUnitStateData data) { @@ -702,8 +811,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, }); updateFuture.thenAccept(r -> { - // Free the old bundle - tombstoneAsync(serviceUnit).thenRun(() -> { + // Delete the old bundle + pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker())).thenRun(() -> { // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); // TODO: Update the load data immediately if needed. @@ -723,6 +832,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, Throwable throwable = FutureUtil.unwrapCompletionException(ex); if ((throwable instanceof MetadataStoreException.BadVersionException) && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) { + log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit", + counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex); pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, bundle, serviceUnit, data, counter, startTime, completionFuture), 100, MILLISECONDS); } else if (throwable instanceof IllegalArgumentException) { @@ -748,8 +859,10 @@ public void handleMetadataSessionEvent(SessionEvent e) { public void handleBrokerRegistrationEvent(String broker, NotificationType type) { if (type == NotificationType.Created) { + log.info("BrokerRegistry detected the broker:{} registry has been created.", broker); handleBrokerCreationEvent(broker); } else if (type == NotificationType.Deleted) { + log.info("BrokerRegistry detected the broker:{} registry has been deleted.", broker); handleBrokerDeletionEvent(broker); } } @@ -769,7 +882,7 @@ private void handleBrokerCreationEvent(String broker) { CompletableFuture future = cleanupJobs.remove(broker); if (future != null) { future.cancel(false); - totalCleanupCancelledCnt++; + totalInactiveBrokerCleanupCancelledCnt++; log.info("Successfully cancelled the ownership cleanup for broker:{}." + " Active cleanup job count:{}", broker, cleanupJobs.size()); @@ -792,7 +905,7 @@ private void handleBrokerDeletionEvent(String broker) { case Stable -> scheduleCleanup(broker, minCleanupDelayTimeInSecs); case Jittery -> scheduleCleanup(broker, maxCleanupDelayTimeInSecs); case Unstable -> { - totalCleanupIgnoredCnt++; + totalInactiveBrokerCleanupIgnoredCnt++; log.error("MetadataState state is unstable. " + "Ignoring the ownership cleanup request for the reported broker :{}", broker); } @@ -803,7 +916,7 @@ private void scheduleCleanup(String broker, long delayInSecs) { cleanupJobs.computeIfAbsent(broker, k -> { Executor delayed = CompletableFuture .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); - totalCleanupScheduledCnt++; + totalInactiveBrokerCleanupScheduledCnt++; return CompletableFuture .runAsync(() -> { try { @@ -821,27 +934,48 @@ private void scheduleCleanup(String broker, long delayInSecs) { broker, delayInSecs, cleanupJobs.size()); } + private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, Set availableBrokers) { + + Optional selectedBroker = brokerSelector.select(availableBrokers, null, getContext()); + if (selectedBroker.isPresent()) { + var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true); + log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", + serviceUnit, orphanData, override); + pubAsync(serviceUnit, override).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}", + serviceUnit, orphanData, override, e); + } + }); + } else { + log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker.", + serviceUnit, orphanData); + } + } + - private void doCleanup(String broker) { + private void doCleanup(String broker) throws ExecutionException, InterruptedException, TimeoutException { long startTime = System.nanoTime(); log.info("Started ownership cleanup for the inactive broker:{}", broker); - int serviceUnitTombstoneCnt = 0; + int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); - for (Map.Entry etr : tableview.entrySet()) { - ServiceUnitStateData stateData = etr.getValue(); - String serviceUnit = etr.getKey(); - if (StringUtils.equals(broker, stateData.broker()) - || StringUtils.equals(broker, stateData.sourceBroker())) { - log.info("Cleaning ownership serviceUnit:{}, stateData:{}.", serviceUnit, stateData); - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " - + "cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart); - } - }); - serviceUnitTombstoneCnt++; + var availableBrokers = new HashSet<>(brokerRegistry.getAvailableBrokersAsync() + .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); + for (var etr : tableview.entrySet()) { + var stateData = etr.getValue(); + var serviceUnit = etr.getKey(); + var state = state(stateData); + if (StringUtils.equals(broker, stateData.broker())) { + if (ServiceUnitState.isInFlightState(state) || state == Owned) { + overrideOwnership(serviceUnit, stateData, availableBrokers); + orphanServiceUnitCleanupCnt++; + } + + } else if (StringUtils.equals(broker, stateData.sourceBroker())) { + if (ServiceUnitState.isInFlightState(state)) { + overrideOwnership(serviceUnit, stateData, availableBrokers); + orphanServiceUnitCleanupCnt++; + } } } @@ -851,28 +985,51 @@ private void doCleanup(String broker) { log.error("Failed to flush the in-flight messages.", e); } - if (serviceUnitTombstoneCnt > 0) { - this.totalCleanupCnt++; - this.totalServiceUnitCleanupTombstoneCnt += serviceUnitTombstoneCnt; - this.totalBrokerCleanupTombstoneCnt++; + if (orphanServiceUnitCleanupCnt > 0) { + this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt; + this.totalInactiveBrokerCleanupCnt++; } double cleanupTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); // TODO: clean load data stores log.info("Completed a cleanup for the inactive broker:{} in {} ms. " - + "Published tombstone for orphan service units: serviceUnitTombstoneCnt:{}, " + + "Cleaned up orphan service units: orphanServiceUnitCleanupCnt:{}, " + "approximate cleanupErrorCnt:{}, metrics:{} ", broker, cleanupTime, - serviceUnitTombstoneCnt, + orphanServiceUnitCleanupCnt, totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), printCleanupMetrics()); cleanupJobs.remove(broker); } - // TODO: integrate this monitor logic when broker registry is added - private void monitorOwnerships(List brokers) { + private Optional getOverrideStateData(String serviceUnit, ServiceUnitStateData orphanData, + Set availableBrokers, + LoadManagerContext context) { + if (isTransferCommand(orphanData)) { + // rollback to the src + return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true)); + } else if (orphanData.state() == Assigning) { // assign + // roll-forward to another broker + Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); + if (selectedBroker.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true)); + } else if (orphanData.state() == Splitting || orphanData.state() == Releasing) { + // rollback to the target broker for split and unload + return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true)); + } else { + var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", + serviceUnit, orphanData); + log.error(msg); + throw new IllegalStateException(msg); + } + } + + @VisibleForTesting + protected void monitorOwnerships(List brokers) { if (!isChannelOwner()) { log.warn("This broker is not the leader now. Skipping ownership monitor"); return; @@ -886,34 +1043,69 @@ private void monitorOwnerships(List brokers) { long startTime = System.nanoTime(); Set inactiveBrokers = new HashSet<>(); Set activeBrokers = new HashSet<>(brokers); - int serviceUnitTombstoneCnt = 0; + Map orphanServiceUnits = new HashMap<>(); + int serviceUnitTombstoneCleanupCnt = 0; + int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); long now = System.currentTimeMillis(); for (Map.Entry etr : tableview.entrySet()) { String serviceUnit = etr.getKey(); ServiceUnitStateData stateData = etr.getValue(); String broker = stateData.broker(); + var state = stateData.state(); if (!activeBrokers.contains(broker)) { inactiveBrokers.add(stateData.broker()); - } else if (stateData.state() != Owned + } else if (state != Owned && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - log.warn("Found long-running orphan(in-flight) serviceUnit:{}, stateData:{}", - serviceUnit, stateData); - - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " - + "cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart); + if (state == Deleted || state == Free) { + if (now - stateData.timestamp() + > semiTerminalStateWaitingTimeInMillis) { + log.info("Found semi-terminal states to tombstone" + + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); + tombstoneAsync(serviceUnit).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " + + "cleanupErrorCnt:{}.", + serviceUnit, stateData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); + serviceUnitTombstoneCleanupCnt++; } - }); - serviceUnitTombstoneCnt++; + } else { + log.warn("Found orphan serviceUnit:{}, stateData:{}", serviceUnit, stateData); + orphanServiceUnits.put(serviceUnit, stateData); + } } } - for (String inactiveBroker : inactiveBrokers) { - handleBrokerDeletionEvent(inactiveBroker); + // Skip cleaning orphan bundles if inactiveBrokers exist. This is a bigger problem. + if (!inactiveBrokers.isEmpty()) { + for (String inactiveBroker : inactiveBrokers) { + handleBrokerDeletionEvent(inactiveBroker); + } + } else if (!orphanServiceUnits.isEmpty()) { + var context = getContext(); + for (var etr : orphanServiceUnits.entrySet()) { + var orphanServiceUnit = etr.getKey(); + var orphanData = etr.getValue(); + var overrideData = getOverrideStateData( + orphanServiceUnit, orphanData, activeBrokers, context); + if (overrideData.isPresent()) { + pubAsync(orphanServiceUnit, overrideData.get()).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, " + + "cleanupErrorCnt:{}.", + orphanServiceUnit, orphanData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); + orphanServiceUnitCleanupCnt++; + } else { + log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{}. will retry..", + orphanServiceUnit, orphanData); + } + } } try { @@ -922,20 +1114,25 @@ private void monitorOwnerships(List brokers) { log.error("Failed to flush the in-flight messages.", e); } - if (serviceUnitTombstoneCnt > 0) { - this.totalServiceUnitCleanupTombstoneCnt += serviceUnitTombstoneCnt; + if (serviceUnitTombstoneCleanupCnt > 0) { + this.totalServiceUnitTombstoneCleanupCnt += serviceUnitTombstoneCleanupCnt; + } + + if (orphanServiceUnitCleanupCnt > 0) { + this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt; } double monitorTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); log.info("Completed the ownership monitor run in {} ms. " - + "Scheduled cleanups for inactiveBrokers:{}. inactiveBrokerCount:{}. " - + "Published tombstone for orphan service units: serviceUnitTombstoneCnt:{}, " - + "approximate cleanupErrorCnt:{}, metrics:{} ", + + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. " + + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. " + + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. " + + "Approximate cleanupErrorCnt:{}, metrics:{}. ", monitorTime, - inactiveBrokers, - inactiveBrokers.size(), - serviceUnitTombstoneCnt, + inactiveBrokers, inactiveBrokers.size(), + orphanServiceUnitCleanupCnt, + serviceUnitTombstoneCleanupCnt, totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), printCleanupMetrics()); @@ -943,17 +1140,19 @@ private void monitorOwnerships(List brokers) { private String printCleanupMetrics() { return String.format( - "{totalCleanupCnt:%d, totalBrokerCleanupTombstoneCnt:%d, " - + "totalServiceUnitCleanupTombstoneCnt:%d, totalCleanupErrorCnt:%d, " - + "totalCleanupScheduledCnt%d, totalCleanupIgnoredCnt:%d, totalCleanupCancelledCnt:%d, " + "{totalInactiveBrokerCleanupCnt:%d, " + + "totalServiceUnitTombstoneCleanupCnt:%d, totalOrphanServiceUnitCleanupCnt:%d, " + + "totalCleanupErrorCnt:%d, " + + "totalInactiveBrokerCleanupScheduledCnt%d, totalInactiveBrokerCleanupIgnoredCnt:%d, " + + "totalInactiveBrokerCleanupCancelledCnt:%d, " + " activeCleanupJobs:%d}", - totalCleanupCnt, - totalBrokerCleanupTombstoneCnt, - totalServiceUnitCleanupTombstoneCnt, + totalInactiveBrokerCleanupCnt, + totalServiceUnitTombstoneCleanupCnt, + totalOrphanServiceUnitCleanupCnt, totalCleanupErrorCnt.get(), - totalCleanupScheduledCnt, - totalCleanupIgnoredCnt, - totalCleanupCancelledCnt, + totalInactiveBrokerCleanupScheduledCnt, + totalInactiveBrokerCleanupIgnoredCnt, + totalInactiveBrokerCleanupCancelledCnt, cleanupJobs.size() ); } @@ -1018,15 +1217,6 @@ public List getMetrics() { } } - - { - var dim = new HashMap<>(dimensions); - dim.put("result", "Total"); - var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupCnt); - metrics.add(metric); - } - { var dim = new HashMap<>(dimensions); dim.put("result", "Failure"); @@ -1039,7 +1229,7 @@ public List getMetrics() { var dim = new HashMap<>(dimensions); dim.put("result", "Skip"); var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupIgnoredCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupIgnoredCnt); metrics.add(metric); } @@ -1047,7 +1237,7 @@ public List getMetrics() { var dim = new HashMap<>(dimensions); dim.put("result", "Cancel"); var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupCancelledCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCancelledCnt); metrics.add(metric); } @@ -1055,13 +1245,14 @@ public List getMetrics() { var dim = new HashMap<>(dimensions); dim.put("result", "Schedule"); var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupScheduledCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupScheduledCnt); metrics.add(metric); } var metric = Metrics.create(dimensions); - metric.put("brk_sunit_state_chn_broker_cleanup_ops_total", totalBrokerCleanupTombstoneCnt); - metric.put("brk_sunit_state_chn_su_cleanup_ops_total", totalServiceUnitCleanupTombstoneCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCnt); + metric.put("brk_sunit_state_chn_orphan_su_cleanup_ops_total", totalOrphanServiceUnitCleanupCnt); + metric.put("brk_sunit_state_chn_su_tombstone_cleanup_ops_total", totalServiceUnitTombstoneCleanupCnt); metrics.add(metric); return metrics; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index 2b21f830dda92..d2a585af9d9d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -18,13 +18,11 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.topics.TopicCompactionStrategy; @@ -50,40 +48,69 @@ public void checkBrokers(boolean check) { @Override public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { - ServiceUnitState prevState = from == null ? Free : from.state(); - ServiceUnitState state = to == null ? Free : to.state(); + if (to == null) { + return false; + } else if (to.force()) { + return false; + } + + + ServiceUnitState prevState = state(from); + ServiceUnitState state = state(to); + if (!ServiceUnitState.isValidTransition(prevState, state)) { return true; } if (checkBrokers) { - if (prevState == Free && (state == Assigned || state == Owned)) { - // Free -> Assigned || Owned broker check - return StringUtils.isBlank(to.broker()); - } else if (prevState == Owned && state == Assigned) { - // Owned -> Assigned(transfer) broker check - return !StringUtils.equals(from.broker(), to.sourceBroker()) - || StringUtils.isBlank(to.broker()) - || StringUtils.equals(from.broker(), to.broker()); - } else if (prevState == Assigned && state == Released) { - // Assigned -> Released(transfer) broker check - return !StringUtils.equals(from.broker(), to.broker()) - || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); - } else if (prevState == Released && state == Owned) { - // Released -> Owned(transfer) broker check - return !StringUtils.equals(from.broker(), to.broker()) - || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); - } else if (prevState == Assigned && state == Owned) { - // Assigned -> Owned broker check - return !StringUtils.equals(from.broker(), to.broker()) - || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); - } else if (prevState == Owned && state == Splitting) { - // Owned -> Splitting broker check - return !StringUtils.equals(from.broker(), to.broker()); + switch (prevState) { + case Owned: + switch (state) { + case Assigning: + return invalidTransfer(from, to); + case Splitting: + case Releasing: + return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + } + case Assigning: + switch (state) { + case Releasing: + return isBlank(to.sourceBroker()) || notEquals(from, to); + case Owned: + return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + } + case Releasing: + switch (state) { + case Owned: + case Free: + return notEquals(from, to); + } + case Splitting: + switch (state) { + case Deleted: + return notEquals(from, to); + } + case Free: + switch (state) { + case Assigning: + return isNotBlank(to.sourceBroker()); + } } } - return false; } -} + private boolean targetNotEquals(ServiceUnitStateData from, ServiceUnitStateData to) { + return !from.broker().equals(to.broker()); + } + + private boolean notEquals(ServiceUnitStateData from, ServiceUnitStateData to) { + return !from.broker().equals(to.broker()) + || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); + } + + private boolean invalidTransfer(ServiceUnitStateData from, ServiceUnitStateData to) { + return !from.broker().equals(to.sourceBroker()) + || from.broker().equals(to.broker()); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index cba459b7875f7..6a04431de64d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -20,24 +20,36 @@ import java.util.Objects; +import org.apache.commons.lang3.StringUtils; /** * Defines data for the service unit state changes. * This data will be broadcast in ServiceUnitStateChannel. */ -public record ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker, long timestamp) { +public record ServiceUnitStateData( + ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp) { public ServiceUnitStateData { Objects.requireNonNull(state); - Objects.requireNonNull(broker); + if (StringUtils.isBlank(broker)) { + throw new IllegalArgumentException("Empty broker"); + } } public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker) { - this(state, broker, sourceBroker, System.currentTimeMillis()); + this(state, broker, sourceBroker, false, System.currentTimeMillis()); } public ServiceUnitStateData(ServiceUnitState state, String broker) { - this(state, broker, null, System.currentTimeMillis()); + this(state, broker, null, false, System.currentTimeMillis()); + } + + public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force) { + this(state, broker, null, force, System.currentTimeMillis()); + } + + public static ServiceUnitState state(ServiceUnitStateData data) { + return data == null ? ServiceUnitState.Init : data.state(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index 9dc4ec649b62b..37b03e275d6bf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -38,7 +38,6 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.CompactionReaderImpl; @@ -63,6 +62,7 @@ public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor { private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class); private static final int MAX_OUTSTANDING = 500; + private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000; private final Duration phaseOneLoopReadTimeout; private final RawBatchMessageContainerImpl batchMessageContainer; @@ -110,7 +110,7 @@ CompletableFuture doCompaction(Reader reader, TopicCompactionStrate if (!(reader instanceof CompactionReaderImpl)) { return CompletableFuture.failedFuture( - new IllegalStateException("reader has to be DelayedAckReaderImpl")); + new IllegalStateException("reader has to be CompactionReaderImpl")); } return reader.hasMessageAvailableAsync() .thenCompose(available -> { @@ -284,9 +284,12 @@ private void phaseOneLoop(Reader reader, CompletableFuture void phaseOneLoop(Reader reader, CompletableFuture void waitForReconnection(Reader reader) { + long started = System.currentTimeMillis(); + + // initial sleep + try { + Thread.sleep(100); + } catch (InterruptedException e) { + } + while (!reader.isConnected()) { + long now = System.currentTimeMillis(); + if (now - started > MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS) { + String errorMsg = String.format( + "Reader has not been reconnected for %d secs. Stopping the compaction.", + MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS / 1000); + log.error(errorMsg); + throw new RuntimeException(errorMsg); + } + log.warn( + "Reader has not been reconnected after the cursor reset. elapsed :{} ms. Retrying " + + "soon.", now - started); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + log.warn("The thread got interrupted while waiting. continuing", e); + } + } + } + private CompletableFuture phaseTwo(PhaseOneResult phaseOneResult, Reader reader, BookKeeper bk) { log.info("Completed phase one. Result:{}. ", phaseOneResult); Map metadata = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 1ef4f660e4af3..001aac34a4ba2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -18,14 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Assign; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Split; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; @@ -53,6 +45,7 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; +import java.util.LinkedHashMap; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -73,6 +66,7 @@ import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; @@ -342,17 +336,19 @@ public void testGetMetrics() throws Exception { FieldUtils.writeDeclaredField(unloadCounter, "loadAvg", 1.5, true); FieldUtils.writeDeclaredField(unloadCounter, "loadStd", 0.3, true); FieldUtils.writeDeclaredField(unloadCounter, "breakdownCounters", Map.of( - Success, Map.of( - Overloaded, new MutableLong(1), - Underloaded, new MutableLong(2)), - Skip, Map.of( - Balanced, new MutableLong(3), - NoBundles, new MutableLong(4), - CoolDown, new MutableLong(5), - OutDatedData, new MutableLong(6), - NoLoadData, new MutableLong(7), - NoBrokers, new MutableLong(8), - Unknown, new MutableLong(9)), + Success, new LinkedHashMap<>() {{ + put(Overloaded, new MutableLong(1)); + put(Underloaded, new MutableLong(2)); + }}, + Skip, new LinkedHashMap<>() {{ + put(Balanced, new MutableLong(3)); + put(NoBundles, new MutableLong(4)); + put(CoolDown, new MutableLong(5)); + put(OutDatedData, new MutableLong(6)); + put(NoLoadData, new MutableLong(7)); + put(NoBrokers, new MutableLong(8)); + put(Unknown, new MutableLong(9)); + }}, Failure, Map.of( Unknown, new MutableLong(10)) ), true); @@ -363,19 +359,24 @@ Unknown, new MutableLong(10)) FieldUtils.readDeclaredField(primaryLoadManager, "splitMetrics", true); SplitCounter splitCounter = new SplitCounter(); FieldUtils.writeDeclaredField(splitCounter, "splitCount", 35l, true); - FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", Map.of( - SplitDecision.Label.Success, Map.of( - Topics, new MutableLong(1), - Sessions, new MutableLong(2), - MsgRate, new MutableLong(3), - Bandwidth, new MutableLong(4), - Admin, new MutableLong(5)), - SplitDecision.Label.Skip, Map.of( + FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", new LinkedHashMap<>() { + { + put(SplitDecision.Label.Success, new LinkedHashMap<>() { + { + put(Topics, new MutableLong(1)); + put(Sessions, new MutableLong(2)); + put(MsgRate, new MutableLong(3)); + put(Bandwidth, new MutableLong(4)); + put(Admin, new MutableLong(5)); + } + }); + put(SplitDecision.Label.Skip, Map.of( SplitDecision.Reason.Balanced, new MutableLong(6) - ), - SplitDecision.Label.Failure, Map.of( - SplitDecision.Reason.Unknown, new MutableLong(7)) - ), true); + )); + put(SplitDecision.Label.Failure, Map.of( + SplitDecision.Reason.Unknown, new MutableLong(7))); + } + }, true); splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress())); } @@ -391,34 +392,39 @@ SplitDecision.Reason.Unknown, new MutableLong(7)) } { - FieldUtils.writeDeclaredField(channel1, "totalCleanupCnt", 1, true); - FieldUtils.writeDeclaredField(channel1, "totalBrokerCleanupTombstoneCnt", 2, true); - FieldUtils.writeDeclaredField(channel1, "totalServiceUnitCleanupTombstoneCnt", 3, true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupErrorCnt", new AtomicLong(4), true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupScheduledCnt", 5, true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupIgnoredCnt", 6, true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupCancelledCnt", 7, true); - FieldUtils.writeDeclaredField(channel1, "ownerLookUpCounters", Map.of( - Owned, new AtomicLong(1), - Assigned, new AtomicLong(2), - Released, new AtomicLong(3), - Splitting, new AtomicLong(4), - Free, new AtomicLong(5) - ), true); - FieldUtils.writeDeclaredField(channel1, "eventCounters", Map.of( - Assign, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(1), new AtomicLong(2)), - Split, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(3), new AtomicLong(4)), - Unload, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(5), new AtomicLong(6)) - ), true); - - FieldUtils.writeDeclaredField(channel1, "handlerCounters", Map.of( - Owned, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(1), new AtomicLong(2)), - Assigned, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(3), new AtomicLong(4)), - Released, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(5), new AtomicLong(6)), - Splitting, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(7), new AtomicLong(8)), - Free, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(9), new AtomicLong(10)) - ), true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCnt", 1, true); + FieldUtils.writeDeclaredField(channel1, "totalServiceUnitTombstoneCleanupCnt", 2, true); + FieldUtils.writeDeclaredField(channel1, "totalOrphanServiceUnitCleanupCnt", 3, true); + FieldUtils.writeDeclaredField(channel1, "totalCleanupErrorCnt", new AtomicLong(4), true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupScheduledCnt", 5, true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupIgnoredCnt", 6, true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCancelledCnt", 7, true); + + Map ownerLookUpCounters = new LinkedHashMap<>(); + Map handlerCounters = new LinkedHashMap<>(); + Map eventCounters = + new LinkedHashMap<>(); + int i = 1; + int j = 0; + for (var state : ServiceUnitState.values()) { + ownerLookUpCounters.put(state, new AtomicLong(i)); + handlerCounters.put(state, + new ServiceUnitStateChannelImpl.Counters( + new AtomicLong(j + 1), new AtomicLong(j + 2))); + i++; + j += 2; + } + i = 0; + for (var type : ServiceUnitStateChannelImpl.EventType.values()) { + eventCounters.put(type, + new ServiceUnitStateChannelImpl.Counters( + new AtomicLong(i + 1), new AtomicLong(i + 2))); + i += 2; + } + FieldUtils.writeDeclaredField(channel1, "ownerLookUpCounters", ownerLookUpCounters, true); + FieldUtils.writeDeclaredField(channel1, "eventCounters", eventCounters, true); + FieldUtils.writeDeclaredField(channel1, "handlerCounters", handlerCounters, true); } var expected = Set.of( @@ -428,55 +434,60 @@ Free, new AtomicLong(5) dimensions=[{broker=localhost, feature=max, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=0.04}] dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_unload_broker_total=2, brk_lb_unload_bundle_total=3}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Failure}], metrics=[{brk_lb_unload_broker_breakdown_total=10}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=CoolDown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=5}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=OutDatedData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=6}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoLoadData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=7}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBrokers, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=8}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=9}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=Underloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=2}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Overloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=1}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Underloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=2}] dimensions=[{broker=localhost, feature=max_ema, metric=bundleUnloading, stat=avg}], metrics=[{brk_lb_resource_usage_stats=1.5}] dimensions=[{broker=localhost, feature=max_ema, metric=bundleUnloading, stat=std}], metrics=[{brk_lb_resource_usage_stats=0.3}] dimensions=[{broker=localhost, metric=bundlesSplit}], metrics=[{brk_lb_bundles_split_total=35}] - dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Topics, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=1}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Sessions, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=2}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=MsgRate, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}] - dimensions=[{broker=localhost, metric=bundlesSplit, reason=Topics, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=1}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Balanced, result=Skip}], metrics=[{brk_lb_bundles_split_breakdown_total=6}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=7}] dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}] - dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}] dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Released}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] - dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}] - dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}] - dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}] - dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}] + dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=6}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=7}] dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=1}] dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=2}] - dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=7}] - dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=8}] - dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}] - dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}] - dimensions=[{broker=localhost, event=Released, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=5}] - dimensions=[{broker=localhost, event=Released, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=6}] - dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=9}] - dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=10}] - dimensions=[{broker=localhost, event=Assigned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}] - dimensions=[{broker=localhost, event=Assigned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=1}] + dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}] + dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}] + dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}] + dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}] + dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}] + dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}] + dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}] + dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=4}] + dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=5}] + dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=6}] + dimensions=[{broker=localhost, event=Assigning, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=7}] + dimensions=[{broker=localhost, event=Assigning, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=8}] + dimensions=[{broker=localhost, event=Releasing, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=9}] + dimensions=[{broker=localhost, event=Releasing, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=10}] + dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=11}] + dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=12}] + dimensions=[{broker=localhost, event=Deleted, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=13}] + dimensions=[{broker=localhost, event=Deleted, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=14}] dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=6}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=7}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=5}] - dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_broker_cleanup_ops_total=2, brk_sunit_state_chn_su_cleanup_ops_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=6}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=7}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1, brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}] """.split("\n")); var actual = primaryLoadManager.getMetrics().stream().map(m -> m.toString()).collect(Collectors.toSet()); assertEquals(actual, expected); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 327afa3cb8891..49eee6ecb7aef 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -18,15 +18,18 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Assign; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Split; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MAX_CLEAN_UP_DELAY_TIME_IN_SECS; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.apache.pulsar.metadata.api.extended.SessionEvent.ConnectionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.Reconnected; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; @@ -52,6 +55,7 @@ import static org.testng.AssertJUnit.assertNotNull; import java.lang.reflect.Field; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -71,8 +75,11 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistryImpl; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; @@ -104,6 +111,12 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private String bundle2; private PulsarTestContext additionalPulsarTestContext; + private LoadManagerContext loadManagerContext; + + private BrokerRegistryImpl registry; + + private BrokerSelectionStrategy brokerSelector; + @BeforeClass @Override protected void setup() throws Exception { @@ -117,11 +130,16 @@ protected void setup() throws Exception { admin.namespaces().createNamespace("public/default"); pulsar1 = pulsar; + registry = new BrokerRegistryImpl(pulsar); + loadManagerContext = mock(LoadManagerContext.class); + brokerSelector = mock(BrokerSelectionStrategy.class); additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf()); pulsar2 = additionalPulsarTestContext.getPulsarService(); - channel1 = spy(new ServiceUnitStateChannelImpl(pulsar1)); + + channel1 = createChannel(pulsar1); channel1.start(); - channel2 = spy(new ServiceUnitStateChannelImpl(pulsar2)); + + channel2 = createChannel(pulsar2); channel2.start(); lookupServiceAddress1 = (String) FieldUtils.readDeclaredField(channel1, "lookupServiceAddress", true); @@ -137,6 +155,8 @@ protected void setup() throws Exception { protected void initTableViews() throws Exception { cleanTableView(channel1, bundle); cleanTableView(channel2, bundle); + cleanOwnershipMonitorCounters(channel1); + cleanOwnershipMonitorCounters(channel2); cleanOpsCounters(channel1); cleanOpsCounters(channel2); } @@ -187,7 +207,7 @@ public void channelOwnerTest() throws Exception { public void channelValidationTest() throws ExecutionException, InterruptedException, IllegalAccessException, PulsarServerException, TimeoutException { - var channel = new ServiceUnitStateChannelImpl(pulsar); + var channel = createChannel(pulsar); int errorCnt = validateChannelStart(channel); assertEquals(6, errorCnt); ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -333,8 +353,8 @@ public void assignmentTest() assertEquals(getOwnerRequests1.size(), 0); assertEquals(getOwnerRequests2.size(), 0); - validateHandlerCounters(channel1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0); - validateHandlerCounters(channel2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0); + validateHandlerCounters(channel1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateHandlerCounters(channel2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); validateEventCounters(channel1, 1, 0, 0, 0, 0, 0); validateEventCounters(channel2, 1, 0, 0, 0, 0, 0); } @@ -379,7 +399,7 @@ public void assignmentTestWhenOneAssignmentFails() } @Test(priority = 4) - public void unloadTest() + public void transferTest() throws ExecutionException, InterruptedException, TimeoutException, IllegalAccessException { var owner1 = channel1.getOwnerAsync(bundle); @@ -409,14 +429,14 @@ public void unloadTest() assertEquals(ownerAddr1, ownerAddr2); assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); - validateHandlerCounters(channel1, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0); - validateHandlerCounters(channel2, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0); + validateHandlerCounters(channel1, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateHandlerCounters(channel2, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); validateEventCounters(channel1, 1, 0, 0, 0, 1, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); } @Test(priority = 5) - public void unloadTestWhenDestBrokerFails() + public void transferTestWhenDestBrokerFails() throws ExecutionException, InterruptedException, IllegalAccessException { var getOwnerRequests1 = getOwnerRequests(channel1); @@ -450,8 +470,8 @@ public void unloadTestWhenDestBrokerFails() Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2)); channel1.publishUnloadEventAsync(unload); // channel1 is broken. the ownership transfer won't be complete. - waitUntilNewState(channel1, bundle); - waitUntilNewState(channel2, bundle); + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); @@ -461,7 +481,7 @@ public void unloadTestWhenDestBrokerFails() assertEquals(1, getOwnerRequests1.size()); assertEquals(1, getOwnerRequests2.size()); - // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. + // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. Awaitility.await().atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); Awaitility.await().atMost(5, TimeUnit.SECONDS) @@ -470,19 +490,42 @@ public void unloadTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); + // recovered, check the monitor update state : Assigned -> Owned + FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); - // TODO: retry lookups and assert that the monitor cleans up the stuck assignments - /* - owner1 = channel1.getOwnerAsync(bundle); - owner2 = channel2.getOwnerAsync(bundle); - assertFalse(channel1.getOwnerAsync(bundle).isDone()); - assertFalse(channel1.getOwnerAsync(bundle).isDone()); - */ FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); - FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + } @Test(priority = 6) @@ -514,11 +557,11 @@ public void splitAndRetryTest() throws Exception { Split split = new Split(bundle, ownerAddr1.get(), new HashMap<>()); channel1.publishSplitEventAsync(split); - waitUntilNewOwner(channel1, bundle, null); - waitUntilNewOwner(channel2, bundle, null); + waitUntilState(channel1, bundle, Deleted); + waitUntilState(channel2, bundle, Deleted); - validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 7, 0); - validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 7, 0); + validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0); + validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0); validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); // Verify the retry count @@ -538,10 +581,49 @@ public void splitAndRetryTest() throws Exception { assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle1).get()); assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle2).get()); + + // try the monitor and check the monitor moves `Deleted` -> `Init` + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + waitUntilState(channel1, bundle, Init); + waitUntilState(channel2, bundle, Init); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 1, + 0, + 0, + 0, + 0, + 0); + cleanTableView(channel1, childBundle1); cleanTableView(channel2, childBundle1); cleanTableView(channel1, childBundle2); cleanTableView(channel2, childBundle2); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); } @Test(priority = 7) @@ -622,6 +704,7 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); + doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); @@ -634,6 +717,10 @@ public void handleBrokerDeletionEventTest() waitUntilNewOwner(channel1, bundle2, broker); waitUntilNewOwner(channel2, bundle2, broker); + channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(lookupServiceAddress2))); + waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + // test stable metadata state leaderChannel.handleMetadataSessionEvent(SessionReestablished); followerChannel.handleMetadataSessionEvent(SessionReestablished); @@ -644,26 +731,30 @@ public void handleBrokerDeletionEventTest() leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); - waitUntilNewOwner(channel1, bundle1, null); - waitUntilNewOwner(channel2, bundle1, null); - waitUntilNewOwner(channel1, bundle2, null); - waitUntilNewOwner(channel2, bundle2, null); + waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); + + assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 1, + 0, + 0); + // test jittery metadata state - channel1.publishAssignEventAsync(bundle1, broker); - channel2.publishAssignEventAsync(bundle2, broker); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -678,13 +769,14 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(1, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 2, + 0, + 0); // broker is back online leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Created); @@ -694,13 +786,14 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 2, + 0, + 1); // broker is offline again @@ -712,35 +805,37 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(1, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(3, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 3, + 0, + 1); // finally cleanup - waitUntilNewOwner(channel1, bundle1, null); - waitUntilNewOwner(channel2, bundle1, null); - waitUntilNewOwner(channel1, bundle2, null); - waitUntilNewOwner(channel2, bundle2, null); + waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(4, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(3, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 2, + 0, + 3, + 0, + 3, + 0, + 1); // test unstable state - channel1.publishAssignEventAsync(bundle1, broker); - channel2.publishAssignEventAsync(bundle2, broker); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -755,13 +850,14 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(4, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(3, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 2, + 0, + 3, + 0, + 3, + 1, + 1); // clean-up FieldUtils.writeDeclaredField(leaderChannel, "maxCleanupDelayTimeInSecs", 3 * 60, true); @@ -774,9 +870,7 @@ public void handleBrokerDeletionEventTest() @Test(priority = 10) public void conflictAndCompactionTest() throws ExecutionException, InterruptedException, TimeoutException, IllegalAccessException, PulsarClientException, PulsarServerException { - - var producer = (Producer) FieldUtils.readDeclaredField(channel1, "producer", true); - producer.newMessage().key(bundle).send(); + String bundle = String.format("%s/%s", "public/default", "0x0000000a_0xffffffff"); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); assertTrue(owner1.get().isEmpty()); @@ -815,7 +909,7 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx .untilAsserted(() -> verify(compactor, times(1)) .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any())); - var channel3 = new ServiceUnitStateChannelImpl(pulsar1); + var channel3 = createChannel(pulsar); channel3.start(); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) @@ -830,10 +924,7 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx @Test(priority = 11) public void ownerLookupCountTests() throws IllegalAccessException { - overrideTableView(channel1, bundle, null); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigned, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1")); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); @@ -842,19 +933,273 @@ public void ownerLookupCountTests() throws IllegalAccessException { channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Released, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1")); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1")); channel1.getOwnerAsync(bundle); - validateOwnerLookUpCounters(channel1, 2, 3, 2, 1, 1); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1")); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1")); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, null); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + validateOwnerLookUpCounters(channel1, 2, 3, 2, 1, 1, 2, 3); + + } + + @Test(priority = 12) + public void unloadTest() + throws ExecutionException, InterruptedException, IllegalAccessException { + + channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + + channel1.publishUnloadEventAsync(unload); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + assertEquals(Optional.empty(), owner1.get()); + assertEquals(Optional.empty(), owner2.get()); + + channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + + ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + Unload unload2 = new Unload(lookupServiceAddress2, bundle, Optional.empty()); + + channel2.publishUnloadEventAsync(unload2); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + + // test monitor if Free -> Init + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + waitUntilState(channel1, bundle, Init); + waitUntilState(channel2, bundle, Init); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 1, + 0, + 0, + 0, + 0, + 0); + + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 30 * 1000, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 300 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + } + + @Test(priority = 13) + public void assignTestWhenDestBrokerFails() + throws ExecutionException, InterruptedException, IllegalAccessException { + + Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + + channel1.publishUnloadEventAsync(unload); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + + assertEquals(Optional.empty(), channel1.getOwnerAsync(bundle).get()); + assertEquals(Optional.empty(), channel2.getOwnerAsync(bundle).get()); + + var producer = (Producer) FieldUtils.readDeclaredField(channel1, + "producer", true); + var spyProducer = spy(producer); + var msg = mock(TypedMessageBuilder.class); + var future = CompletableFuture.failedFuture(new RuntimeException()); + doReturn(msg).when(spyProducer).newMessage(); + doReturn(msg).when(msg).key(any()); + doReturn(msg).when(msg).value(any()); + doReturn(future).when(msg).sendAsync(); + FieldUtils.writeDeclaredField(channel2, "producer", spyProducer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any()); + channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); + // channel1 is broken. the assign won't be complete. + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + assertFalse(owner1.isDone()); + assertFalse(owner2.isDone()); + + // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. + Awaitility.await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); + Awaitility.await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); + + // recovered, check the monitor update state : Assigned -> Owned + FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); } + @Test(priority = 14) + public void splitTestWhenDestBrokerFails() + throws ExecutionException, InterruptedException, IllegalAccessException { + + + Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + + channel1.publishUnloadEventAsync(unload); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + + channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + + waitUntilState(channel1, bundle, Owned); + waitUntilState(channel2, bundle, Owned); + + assertEquals(lookupServiceAddress1, channel1.getOwnerAsync(bundle).get().get()); + assertEquals(lookupServiceAddress1, channel2.getOwnerAsync(bundle).get().get()); + + var producer = (Producer) FieldUtils.readDeclaredField(channel1, + "producer", true); + var spyProducer = spy(producer); + var msg = mock(TypedMessageBuilder.class); + var future = CompletableFuture.failedFuture(new RuntimeException()); + doReturn(msg).when(spyProducer).newMessage(); + doReturn(msg).when(msg).key(any()); + doReturn(msg).when(msg).value(any()); + doReturn(future).when(msg).sendAsync(); + FieldUtils.writeDeclaredField(channel1, "producer", spyProducer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + channel2.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, null)); + // channel1 is broken. the split won't be complete. + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + + // recovered, check the monitor update state : Splitting -> Owned + FieldUtils.writeDeclaredField(channel1, "producer", producer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); - // TODO: add the channel recovery test when broker registry is added. + } private static ConcurrentOpenHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { @@ -926,7 +1271,7 @@ private static void waitUntilNewOwner(ServiceUnitStateChannel channel, String se }); } - private static void waitUntilNewState(ServiceUnitStateChannel channel, String key) + private static void waitUntilState(ServiceUnitStateChannel channel, String key) throws IllegalAccessException { TableViewImpl tv = (TableViewImpl) FieldUtils.readField(channel, "tableview", true); @@ -943,6 +1288,20 @@ private static void waitUntilNewState(ServiceUnitStateChannel channel, String ke }); } + private static void waitUntilState(ServiceUnitStateChannel channel, String key, ServiceUnitState expected) + throws IllegalAccessException { + TableViewImpl tv = (TableViewImpl) + FieldUtils.readField(channel, "tableview", true); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .until(() -> { // wait until true + ServiceUnitStateData data = tv.get(key); + ServiceUnitState actual = state(data); + return actual == expected; + }); + } + private static void cleanTableView(ServiceUnitStateChannel channel, String serviceUnit) throws IllegalAccessException { var tv = (TableViewImpl) @@ -994,6 +1353,16 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel) } } + private void cleanOwnershipMonitorCounters(ServiceUnitStateChannel channel) throws IllegalAccessException { + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalServiceUnitTombstoneCleanupCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalOrphanServiceUnitCleanupCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalCleanupErrorCnt", new AtomicLong(0), true); + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupScheduledCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupIgnoredCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupCancelledCnt", 0, true); + } + private static long getCleanupMetric(ServiceUnitStateChannel channel, String metric) throws IllegalAccessException { Object var = FieldUtils.readDeclaredField(channel, metric, true); @@ -1009,7 +1378,9 @@ private static void validateHandlerCounters(ServiceUnitStateChannel channel, long ownedT, long ownedF, long releasedT, long releasedF, long splittingT, long splittingF, - long freeT, long freeF) + long freeT, long freeF, + long initT, long initF, + long deletedT, long deletedF) throws IllegalAccessException { var handlerCounters = (Map) @@ -1019,16 +1390,20 @@ private static void validateHandlerCounters(ServiceUnitStateChannel channel, .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { // wait until true - assertEquals(assignedT, handlerCounters.get(Assigned).getTotal().get()); - assertEquals(assignedF, handlerCounters.get(Assigned).getFailure().get()); + assertEquals(assignedT, handlerCounters.get(Assigning).getTotal().get()); + assertEquals(assignedF, handlerCounters.get(Assigning).getFailure().get()); assertEquals(ownedT, handlerCounters.get(Owned).getTotal().get()); assertEquals(ownedF, handlerCounters.get(Owned).getFailure().get()); - assertEquals(releasedT, handlerCounters.get(Released).getTotal().get()); - assertEquals(releasedF, handlerCounters.get(Released).getFailure().get()); + assertEquals(releasedT, handlerCounters.get(Releasing).getTotal().get()); + assertEquals(releasedF, handlerCounters.get(Releasing).getFailure().get()); assertEquals(splittingT, handlerCounters.get(Splitting).getTotal().get()); assertEquals(splittingF, handlerCounters.get(Splitting).getFailure().get()); assertEquals(freeT, handlerCounters.get(Free).getTotal().get()); assertEquals(freeF, handlerCounters.get(Free).getFailure().get()); + assertEquals(initT, handlerCounters.get(Init).getTotal().get()); + assertEquals(initF, handlerCounters.get(Init).getFailure().get()); + assertEquals(deletedT, handlerCounters.get(Deleted).getTotal().get()); + assertEquals(deletedF, handlerCounters.get(Deleted).getFailure().get()); }); } @@ -1059,7 +1434,10 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel, long owned, long released, long splitting, - long free) + long free, + long deleted, + long init + ) throws IllegalAccessException { var ownerLookUpCounters = (Map) @@ -1069,11 +1447,48 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel, .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { // wait until true - assertEquals(assigned, ownerLookUpCounters.get(Assigned).get()); + assertEquals(assigned, ownerLookUpCounters.get(Assigning).get()); assertEquals(owned, ownerLookUpCounters.get(Owned).get()); - assertEquals(released, ownerLookUpCounters.get(Released).get()); + assertEquals(released, ownerLookUpCounters.get(Releasing).get()); assertEquals(splitting, ownerLookUpCounters.get(Splitting).get()); assertEquals(free, ownerLookUpCounters.get(Free).get()); + assertEquals(deleted, ownerLookUpCounters.get(Deleted).get()); + assertEquals(init, ownerLookUpCounters.get(Init).get()); }); } + + private static void validateMonitorCounters(ServiceUnitStateChannel channel, + long totalInactiveBrokerCleanupCnt, + long totalServiceUnitTombstoneCleanupCnt, + long totalOrphanServiceUnitCleanupCnt, + long totalCleanupErrorCnt, + long totalInactiveBrokerCleanupScheduledCnt, + long totalInactiveBrokerCleanupIgnoredCnt, + long totalInactiveBrokerCleanupCancelledCnt) + throws IllegalAccessException { + assertEquals(totalInactiveBrokerCleanupCnt, getCleanupMetric(channel, "totalInactiveBrokerCleanupCnt")); + assertEquals(totalServiceUnitTombstoneCleanupCnt, + getCleanupMetric(channel, "totalServiceUnitTombstoneCleanupCnt")); + assertEquals(totalOrphanServiceUnitCleanupCnt, getCleanupMetric(channel, "totalOrphanServiceUnitCleanupCnt")); + assertEquals(totalCleanupErrorCnt, getCleanupMetric(channel, "totalCleanupErrorCnt")); + assertEquals(totalInactiveBrokerCleanupScheduledCnt, + getCleanupMetric(channel, "totalInactiveBrokerCleanupScheduledCnt")); + assertEquals(totalInactiveBrokerCleanupIgnoredCnt, + getCleanupMetric(channel, "totalInactiveBrokerCleanupIgnoredCnt")); + assertEquals(totalInactiveBrokerCleanupCancelledCnt, + getCleanupMetric(channel, "totalInactiveBrokerCleanupCancelledCnt")); + } + + ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) + throws IllegalAccessException { + var tmpChannel = new ServiceUnitStateChannelImpl(pulsar); + FieldUtils.writeDeclaredField(tmpChannel, "ownershipMonitorDelayTimeInSecs", 5, true); + var channel = spy(tmpChannel); + + doReturn(loadManagerContext).when(channel).getContext(); + doReturn(registry).when(channel).getBrokerRegistry(); + doReturn(brokerSelector).when(channel).getBrokerSelector(); + + return channel; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 49b55f7660a81..1a4aba15f9e6f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.testng.Assert.assertTrue; @@ -36,7 +38,7 @@ ServiceUnitStateData data(ServiceUnitState state) { } ServiceUnitStateData data(ServiceUnitState state, String dst) { - return new ServiceUnitStateData(state, dst, "broker"); + return new ServiceUnitStateData(state, dst, null); } ServiceUnitStateData data(ServiceUnitState state, String src, String dst) { return new ServiceUnitStateData(state, dst, src); @@ -45,46 +47,95 @@ ServiceUnitStateData data(ServiceUnitState state, String src, String dst) { @Test public void test() throws InterruptedException { String dst = "dst"; - assertTrue(strategy.shouldKeepLeft(data(Free), data(Free))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Assigned, ""))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Owned, ""))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Released))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Splitting))); + String src = "src"; + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Init, dst), + new ServiceUnitStateData(Init, dst, true))); + + assertFalse(strategy.shouldKeepLeft( + data(Owned), null)); + + assertTrue(strategy.shouldKeepLeft(data(Init), data(Init))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Free))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Assigning))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Owned))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Releasing))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Splitting))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Deleted))); - assertFalse(strategy.shouldKeepLeft(data(Assigned), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Assigned), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "dst2"), data(Owned, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "src1", dst), data(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigned), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "dst2"), data(Released, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "src1", dst), data(Released, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigned, dst), data(Released, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigned), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, src, dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data(Releasing, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Assigned, ""))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Assigned, "src", dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned), data(Assigned, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, "dst1"))); assertTrue(strategy.shouldKeepLeft(data(Owned), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Released))); - assertTrue(strategy.shouldKeepLeft(data(Owned,"dst2"), data(Splitting, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Releasing, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Releasing, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Splitting, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Splitting, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Splitting, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Released), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Released), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Released, "dst2"), data(Owned, dst))); - assertTrue(strategy.shouldKeepLeft(data(Released, "src1", dst), data(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Released), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Released), data(Released))); - assertTrue(strategy.shouldKeepLeft(data(Released), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Init))); + assertFalse(strategy.shouldKeepLeft(data(Releasing), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Free, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Free, "src2", dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data(Owned, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Owned, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data(Owned, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Splitting), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Assigned))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Assigning))); assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Released))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Releasing))); assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, src, "dst1"), data(Deleted, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "dst1"), data(Deleted, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "src1", dst), data(Deleted, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, dst), data(Deleted, dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, src, dst), data(Deleted, src, dst))); + + assertFalse(strategy.shouldKeepLeft(data(Deleted), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Deleted))); + + assertFalse(strategy.shouldKeepLeft(data(Free), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Free))); + assertFalse(strategy.shouldKeepLeft(data(Free), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Deleted))); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java index 7b9afee9ce2d1..9617c8a8c2bd0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; @@ -41,8 +41,8 @@ public void testConstructors() throws InterruptedException { Thread.sleep(10); - ServiceUnitStateData data2 = new ServiceUnitStateData(Assigned, "A", "B"); - assertEquals(data2.state(), Assigned); + ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B"); + assertEquals(data2.state(), Assigning); assertEquals(data2.broker(), "A"); assertEquals(data2.sourceBroker(), "B"); assertThat(data2.timestamp()).isGreaterThan(data1.timestamp()); @@ -53,15 +53,20 @@ public void testNullState() { new ServiceUnitStateData(null, "A"); } - @Test(expectedExceptions = NullPointerException.class) + @Test(expectedExceptions = IllegalArgumentException.class) public void testNullBroker() { new ServiceUnitStateData(Owned, null); } + @Test(expectedExceptions = IllegalArgumentException.class) + public void testEmptyBroker() { + new ServiceUnitStateData(Owned, ""); + } + @Test public void jsonWriteAndReadTest() throws JsonProcessingException { ObjectMapper mapper = ObjectMapperFactory.create(); - final ServiceUnitStateData src = new ServiceUnitStateData(Assigned, "A", "B"); + final ServiceUnitStateData src = new ServiceUnitStateData(Assigning, "A", "B"); String json = mapper.writeValueAsString(src); ServiceUnitStateData dst = mapper.readValue(json, ServiceUnitStateData.class); assertEquals(dst, src); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java index 69e6a2d204c0e..f5f1fe7bc575f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -29,39 +31,75 @@ @Test(groups = "broker") public class ServiceUnitStateTest { + @Test + public void testInFlights() { + assertFalse(ServiceUnitState.isInFlightState(Init)); + assertFalse(ServiceUnitState.isInFlightState(Free)); + assertFalse(ServiceUnitState.isInFlightState(Owned)); + assertTrue(ServiceUnitState.isInFlightState(Assigning)); + assertTrue(ServiceUnitState.isInFlightState(Releasing)); + assertTrue(ServiceUnitState.isInFlightState(Splitting)); + assertFalse(ServiceUnitState.isInFlightState(Deleted)); + } @Test public void testTransitions() { + assertFalse(ServiceUnitState.isValidTransition(Init, Init)); + assertTrue(ServiceUnitState.isValidTransition(Init, Free)); + assertTrue(ServiceUnitState.isValidTransition(Init, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Init, Assigning)); + assertTrue(ServiceUnitState.isValidTransition(Init, Releasing)); + assertTrue(ServiceUnitState.isValidTransition(Init, Splitting)); + assertTrue(ServiceUnitState.isValidTransition(Init, Deleted)); + + assertTrue(ServiceUnitState.isValidTransition(Free, Init)); assertFalse(ServiceUnitState.isValidTransition(Free, Free)); - assertTrue(ServiceUnitState.isValidTransition(Free, Assigned)); - assertTrue(ServiceUnitState.isValidTransition(Free, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Free, Released)); - assertTrue(ServiceUnitState.isValidTransition(Free, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Free, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Free, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Free, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Free, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Free, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Assigned, Free)); - assertFalse(ServiceUnitState.isValidTransition(Assigned, Assigned)); - assertTrue(ServiceUnitState.isValidTransition(Assigned, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Assigned, Released)); - assertFalse(ServiceUnitState.isValidTransition(Assigned, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Init)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Free)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning)); + assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Assigning, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Free)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Assigned)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Init)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Free)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Assigning)); assertFalse(ServiceUnitState.isValidTransition(Owned, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Owned, Released)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing)); assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Released, Free)); - assertFalse(ServiceUnitState.isValidTransition(Released, Assigned)); - assertTrue(ServiceUnitState.isValidTransition(Released, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Released, Released)); - assertFalse(ServiceUnitState.isValidTransition(Released, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Init)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Free)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Assigning)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Owned)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Splitting, Free)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigned)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Init)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Free)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigning)); assertFalse(ServiceUnitState.isValidTransition(Splitting, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Released)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Releasing)); assertFalse(ServiceUnitState.isValidTransition(Splitting, Splitting)); + assertTrue(ServiceUnitState.isValidTransition(Splitting, Deleted)); + + assertTrue(ServiceUnitState.isValidTransition(Deleted, Init)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Free)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Owned)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Deleted)); } } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 41eaa640d28db..4c1d4f7d2a89d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.compaction; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -42,7 +45,9 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.bookkeeper.client.BookKeeper; import org.apache.commons.lang.reflect.FieldUtils; @@ -81,55 +86,36 @@ public class ServiceUnitStateCompactionTest extends MockedPulsarServiceBaseTest private Schema schema; private ServiceUnitStateCompactionStrategy strategy; - private ServiceUnitState testState0 = Free; - private ServiceUnitState testState1 = Free; - private ServiceUnitState testState2 = Free; - private ServiceUnitState testState3 = Free; - private ServiceUnitState testState4 = Free; + private ServiceUnitState testState = Init; private static Random RANDOM = new Random(); private ServiceUnitStateData testValue(ServiceUnitState state, String broker) { - if (state == Free) { + if (state == Init) { return null; } return new ServiceUnitStateData(state, broker); } - private ServiceUnitStateData testValue0(String broker) { - ServiceUnitState to = nextValidState(testState0); - testState0 = to; + private ServiceUnitStateData testValue(String broker) { + ServiceUnitState to = nextValidStateNonSplit(testState); + testState = to; return testValue(to, broker); } - private ServiceUnitStateData testValue1(String broker) { - ServiceUnitState to = nextValidState(testState1); - testState1 = to; - return testValue(to, broker); - } - - private ServiceUnitStateData testValue2(String broker) { - ServiceUnitState to = nextValidState(testState2); - testState2 = to; - return testValue(to, broker); - } - - private ServiceUnitStateData testValue3(String broker) { - ServiceUnitState to = nextValidState(testState3); - testState3 = to; - return testValue(to, broker); - } - - private ServiceUnitStateData testValue4(String broker) { - ServiceUnitState to = nextValidState(testState4); - testState4 = to; - return testValue(to, broker); + private ServiceUnitState nextValidState(ServiceUnitState from) { + List candidates = Arrays.stream(ServiceUnitState.values()) + .filter(to -> isValidTransition(from, to)) + .collect(Collectors.toList()); + var state= candidates.get(RANDOM.nextInt(candidates.size())); + return state; } - private ServiceUnitState nextValidState(ServiceUnitState from) { + private ServiceUnitState nextValidStateNonSplit(ServiceUnitState from) { List candidates = Arrays.stream(ServiceUnitState.values()) - .filter(to -> to != Free && to != Splitting && isValidTransition(from, to)) + .filter(to -> to != Init && to != Splitting && to != Deleted + && isValidTransition(from, to)) .collect(Collectors.toList()); var state= candidates.get(RANDOM.nextInt(candidates.size())); return state; @@ -140,23 +126,11 @@ private ServiceUnitState nextInvalidState(ServiceUnitState from) { .filter(to -> !isValidTransition(from, to)) .collect(Collectors.toList()); if (candidates.size() == 0) { - return null; + return Init; } return candidates.get(RANDOM.nextInt(candidates.size())); } - private List nextStatesToNull(ServiceUnitState from) { - if (from == null) { - return List.of(); - } - return switch (from) { - case Assigned -> List.of(Owned); - case Owned -> List.of(); - case Splitting -> List.of(); - default -> List.of(); - }; - } - @BeforeMethod @Override public void setup() throws Exception { @@ -174,6 +148,7 @@ public void setup() throws Exception { strategy = new ServiceUnitStateCompactionStrategy(); strategy.checkBrokers(false); + testState = Init; } @@ -222,10 +197,21 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException { int keyIndex = r.nextInt(maxKeys); String key = "key" + keyIndex; ServiceUnitStateData prev = expected.get(key); - ServiceUnitState prevState = prev == null ? Free : prev.state(); - ServiceUnitState state = r.nextBoolean() ? nextInvalidState(prevState) : + ServiceUnitState prevState = state(prev); + boolean invalid = r.nextBoolean(); + ServiceUnitState state = invalid ? nextInvalidState(prevState) : nextValidState(prevState); - ServiceUnitStateData value = new ServiceUnitStateData(state, key + ":" + j); + ServiceUnitStateData value; + if (invalid) { + value = new ServiceUnitStateData(state, key + ":" + j, false); + } else { + if (state == Init) { + value = new ServiceUnitStateData(state, key + ":" + j, true); + } else { + value = new ServiceUnitStateData(state, key + ":" + j, false); + } + } + producer.newMessage().key(key).value(value).send(); if (!strategy.shouldKeepLeft(prev, value)) { expected.put(key, value); @@ -387,24 +373,26 @@ public void testReadCompactedBeforeCompaction() throws Exception { .create(); pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); - - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + String key = "key0"; + var testValues = Arrays.asList( + testValue("content0"), testValue("content1"), testValue("content2")); + for (var val : testValues) { + producer.newMessage().key(key).value(val).send(); + } try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content0"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(0)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content1"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(1)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } StrategicTwoPhaseCompactor compactor @@ -414,8 +402,8 @@ public void testReadCompactedBeforeCompaction() throws Exception { try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } } @@ -430,30 +418,37 @@ public void testReadEntriesAfterCompaction() throws Exception { pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + String key = "key0"; + var testValues = Arrays.asList( + testValue( "content0"), + testValue("content1"), + testValue( "content2"), + testValue("content3")); + producer.newMessage().key(key).value(testValues.get(0)).send(); + producer.newMessage().key(key).value(testValues.get(1)).send(); + producer.newMessage().key(key).value(testValues.get(2)).send(); StrategicTwoPhaseCompactor compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); compactor.compact(topic, strategy).get(); - producer.newMessage().key("key0").value(testValue0("content3")).send(); + producer.newMessage().key(key).value(testValues.get(3)).send(); try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content3"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(3)); } } @Test public void testSeekEarliestAfterCompaction() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; Producer producer = pulsarClient.newProducer(schema) @@ -461,9 +456,14 @@ public void testSeekEarliestAfterCompaction() throws Exception { .enableBatching(true) .create(); - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + String key = "key0"; + var testValues = Arrays.asList( + testValue("content0"), + testValue("content1"), + testValue("content2")); + for (var val : testValues) { + producer.newMessage().key(key).value(val).send(); + } StrategicTwoPhaseCompactor compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); @@ -473,8 +473,8 @@ public void testSeekEarliestAfterCompaction() throws Exception { .readCompacted(true).subscribe()) { consumer.seek(MessageId.earliest); Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") @@ -482,34 +482,153 @@ public void testSeekEarliestAfterCompaction() throws Exception { consumer.seek(MessageId.earliest); Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content0"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(0)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content1"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(1)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } } @Test - public void testBrokerRestartAfterCompaction() throws Exception { + public void testSlowTableviewAfterCompaction() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + String strategyClassName = "topicCompactionStrategyClassName"; + strategy.checkBrokers(true); + + pulsarClient.newConsumer(schema) + .topic(topic) + .subscriptionName("sub1") + .readCompacted(true) + .subscribe().close(); + + var fastTV = pulsar.getClient().newTableViewBuilder(schema) + .topic(topic) + .subscriptionName("fastTV") + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + + var defaultConf = getDefaultConf(); + var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); + var pulsar2 = additionalPulsarTestContext.getPulsarService(); + + var slowTV = pulsar2.getClient().newTableViewBuilder(schema) + .topic(topic) + .subscriptionName("slowTV") + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + + var semaphore = new Semaphore(0); + AtomicBoolean handledReleased = new AtomicBoolean(false); + + slowTV.listen((k, v) -> { + if (v.state() == Assigning) { + try { + // Stuck at handling Assigned + handledReleased.set(false); + semaphore.acquire(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } else if (v.state() == Releasing) { + handledReleased.set(true); + } + }); + + // Configure retention to ensue data is retained for reader + admin.namespaces().setRetention("my-property/use/my-ns", + new RetentionPolicies(-1, -1)); Producer producer = pulsarClient.newProducer(schema) .topic(topic) .enableBatching(true) + .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); - pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); + StrategicTwoPhaseCompactor compactor + = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + String bundle = "bundle1"; + String src = "broker0"; + String dst = "broker1"; + producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src)).send(); + for (int i = 0; i < 3; i++) { + var assignedStateData = new ServiceUnitStateData(Assigning, dst, src); + producer.newMessage().key(bundle).value(assignedStateData).send(); + producer.newMessage().key(bundle).value(assignedStateData).send(); + var releasedStateData = new ServiceUnitStateData(Releasing, dst, src); + producer.newMessage().key(bundle).value(releasedStateData).send(); + producer.newMessage().key(bundle).value(releasedStateData).send(); + var ownedStateData = new ServiceUnitStateData(Owned, dst, src); + producer.newMessage().key(bundle).value(ownedStateData).send(); + producer.newMessage().key(bundle).value(ownedStateData).send(); + compactor.compact(topic, strategy).get(); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(fastTV.get(bundle), ownedStateData)); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(slowTV.get(bundle), assignedStateData)); + assertTrue(!handledReleased.get()); + semaphore.release(); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(slowTV.get(bundle), ownedStateData)); + + var newTv = pulsar.getClient().newTableView(schema) + .topic(topic) + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(newTv.get(bundle), ownedStateData)); + + src = dst; + dst = "broker" + (i + 2); + newTv.close(); + } + + producer.close(); + slowTV.close(); + fastTV.close(); + pulsar2.close(); + + } - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + @Test + public void testBrokerRestartAfterCompaction() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + + Producer producer = pulsarClient.newProducer(schema) + .topic(topic) + .enableBatching(true) + .create(); + String key = "key0"; + pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); + var testValues = Arrays.asList( + testValue("content0"), testValue("content1"), testValue("content2")); + for (var val : testValues) { + producer.newMessage().key(key).value(val).send(); + } StrategicTwoPhaseCompactor compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); compactor.compact(topic, strategy).get(); @@ -517,8 +636,8 @@ public void testBrokerRestartAfterCompaction() throws Exception { try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(testValues.size() - 1)); } stopBroker(); @@ -534,8 +653,8 @@ public void testBrokerRestartAfterCompaction() throws Exception { try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(testValues.size() - 1)); } } @@ -554,13 +673,14 @@ public void testCompactEmptyTopic() throws Exception { = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); compactor.compact(topic, strategy).get(); - producer.newMessage().key("key0").value(testValue0( "content0")).send(); + var testValue = testValue( "content0"); + producer.newMessage().key("key0").value(testValue).send(); try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content0"); + Assert.assertEquals(m.getValue(), testValue); } } @@ -583,10 +703,10 @@ public void testWholeBatchCompactedOut() throws Exception { .batchingMaxPublishDelay(1, TimeUnit.HOURS) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create()) { - producerBatch.newMessage().key("key1").value(testValue1("my-message-1")).sendAsync(); - producerBatch.newMessage().key("key1").value(testValue1( "my-message-2")).sendAsync(); - producerBatch.newMessage().key("key1").value(testValue1("my-message-3")).sendAsync(); - producerNormal.newMessage().key("key1").value(testValue1( "my-message-4")).send(); + producerBatch.newMessage().key("key1").value(testValue("my-message-1")).sendAsync(); + producerBatch.newMessage().key("key1").value(testValue( "my-message-2")).sendAsync(); + producerBatch.newMessage().key("key1").value(testValue("my-message-3")).sendAsync(); + producerNormal.newMessage().key("key1").value(testValue( "my-message-4")).send(); } // compact the topic @@ -610,9 +730,9 @@ public void testCompactionWithLastDeletedKey() throws Exception { pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); - producer.newMessage().key("1").value(testValue(Owned, "1")).send(); - producer.newMessage().key("2").value(testValue(Owned, "3")).send(); - producer.newMessage().key("3").value(testValue(Owned, "5")).send(); + producer.newMessage().key("1").value(testValue("1")).send(); + producer.newMessage().key("2").value(testValue("3")).send(); + producer.newMessage().key("3").value(testValue( "5")).send(); producer.newMessage().key("1").value(null).send(); producer.newMessage().key("2").value(null).send(); @@ -707,7 +827,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() List> futures = new ArrayList<>(messages); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); @@ -720,7 +840,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() // 3. Send more ten messages futures.clear(); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + 10 + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + 10 + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); @@ -754,7 +874,7 @@ public void testReadUnCompacted() List> futures = new ArrayList<>(messages); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); @@ -767,7 +887,7 @@ public void testReadUnCompacted() // 3. Send more ten messages futures.clear(); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + 10 + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + 10 + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); try (Consumer consumer = pulsarClient.newConsumer(schema) @@ -788,9 +908,6 @@ public void testReadUnCompacted() } // 4.Send empty message to delete the key-value in the compacted topic. - for (ServiceUnitState state : nextStatesToNull(testState0)) { - producer.newMessage().key(key).value(new ServiceUnitStateData(state, "xx")).send(); - } producer.newMessage().key(key).value(null).send(); // 5.compact the topic. @@ -807,7 +924,7 @@ public void testReadUnCompacted() } for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + 20 + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + 20 + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); From 870334029119dce41f2a6e986db5ca46d023220f Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Fri, 24 Feb 2023 02:14:42 +0200 Subject: [PATCH 011/404] [improve][doc] Changing subject prefix for PIPs in the mailing list (#19617) --- wiki/proposals/PIP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index afd0eac5fdfd8..d972956df0276 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -80,7 +80,7 @@ The process works in the following way: 1. The author(s) of the proposal will create a GitHub issue ticket choosing the template for PIP proposals. 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list - to start the discussion, using subject prefix `[PIP] xxx`. The discussion + to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: `. The discussion need to happen in the mailing list. Please avoid discussing it using GitHub comments in the PIP GitHub issue, as it creates two tracks of feedback. @@ -147,4 +147,4 @@ If there are alternatives that were already considered by the authors or, after the discussion, by the community, and were rejected, please list them here along with the reason why they were rejected. -``` \ No newline at end of file +``` From 3b075a60f04938fa1a90acc2fd856168ca0cadef Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Fri, 24 Feb 2023 02:15:15 +0200 Subject: [PATCH 012/404] [improve][doc] Clarify where to grab the number for the PIP (#19610) --- wiki/proposals/PIP.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index d972956df0276..f76c9f0f7a235 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -78,7 +78,9 @@ A PIP proposal can be in these states: The process works in the following way: 1. The author(s) of the proposal will create a GitHub issue ticket choosing the - template for PIP proposals. + template for PIP proposals. The issue title should be "PIP-xxx: title", where + the "xxx" number should be chosen to be the next number from the existing PIP + issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)). 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: `. The discussion need to happen in the mailing list. Please avoid discussing it using @@ -89,12 +91,12 @@ The process works in the following way: 4. Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list. Everyone - is welcome to vote on the proposal, though it will considered to be binding + is welcome to vote on the proposal, though it will be considered to be binding only the vote of PMC members. I would be required to have a lazy majority of at least 3 binding +1s votes. The vote should stay open for at least 48 hours. 5. When the vote is closed, if the outcome is positive, the state of the - proposal is updated and the Pull Requests associated with this proposal can + proposal is updated, and the Pull Requests associated with this proposal can start to get merged into the master branch. All the Pull Requests that are created, should always reference the From bf982f4995e624659021191982f7fedc13fc3ba0 Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Thu, 23 Feb 2023 18:44:09 -0800 Subject: [PATCH 013/404] [improve] configure whether function consumer should skip to latest (#17214) --- .../org/apache/pulsar/common/functions/FunctionConfig.java | 2 ++ .../java/org/apache/pulsar/admin/cli/CmdFunctions.java | 7 +++++++ .../pulsar/functions/instance/JavaInstanceRunnable.java | 5 +++++ .../pulsar/functions/source/MultiConsumerPulsarSource.java | 6 ++++++ .../apache/pulsar/functions/source/PulsarSourceConfig.java | 3 ++- .../functions/source/SingleConsumerPulsarSource.java | 6 ++++++ pulsar-functions/proto/src/main/proto/Function.proto | 1 + .../apache/pulsar/functions/utils/FunctionConfigUtils.java | 6 ++++++ 8 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java index 0b26e7e93b5f0..e304f25d5d373 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java @@ -131,6 +131,8 @@ public enum Runtime { private Integer maxPendingAsyncRequests; // Whether the pulsar admin client exposed to function context, default is disabled. private Boolean exposePulsarAdminClientEnabled; + // Whether the consumer should skip to latest position in case of failure recovery + private Boolean skipToLatest; @Builder.Default private SubscriptionInitialPosition subscriptionPosition = SubscriptionInitialPosition.Latest; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index bc2585bc67bc9..05bab9c6f198b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -325,6 +325,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--subs-position", description = "Pulsar source subscription position if user wants to " + "consume messages from the specified location #Java") protected SubscriptionInitialPosition subsPosition; + @Parameter(names = "--skip-to-latest", description = "Whether or not the consumer skip to latest message " + + "upon function instance restart", arity = 1) + protected Boolean skipToLatest; @Parameter(names = "--parallelism", description = "The parallelism factor of a Pulsar Function " + "(i.e. the number of function instances to run) #Java") protected Integer parallelism; @@ -548,6 +551,10 @@ void processArguments() throws Exception { functionConfig.setSubscriptionPosition(subsPosition); } + if (null != skipToLatest) { + functionConfig.setSkipToLatest(skipToLatest); + } + if (null != userConfigString) { Type type = new TypeToken>() {}.getType(); Map userConfigMap = new Gson().fromJson(userConfigString, type); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 5f82e93470089..0dbfa0945caa7 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -789,7 +789,12 @@ private void setupInput(ContextImpl contextImpl) throws Exception { convertFromFunctionDetailsSubscriptionPosition(sourceSpec.getSubscriptionPosition()) ); + pulsarSourceConfig.setSkipToLatest( + sourceSpec.getSkipToLatest() + ); + Objects.requireNonNull(contextImpl.getSubscriptionType()); + pulsarSourceConfig.setSubscriptionType(contextImpl.getSubscriptionType()); pulsarSourceConfig.setTypeClassName(sourceSpec.getTypeClassName()); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java index 61f5bfacb35a7..533e8d42c11ac 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -66,6 +67,11 @@ public void open(Map config, SourceContext sourceContext) throws cb.messageListener(this); Consumer consumer = cb.subscribeAsync().join(); + + if (pulsarSourceConfig.getSkipToLatest() != null && pulsarSourceConfig.getSkipToLatest()) { + consumer.seek(MessageId.latest); + } + inputConsumers.add(consumer); } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java index 5315702d7c0e4..ad6ea5a877f30 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java @@ -30,7 +30,8 @@ public abstract class PulsarSourceConfig { SubscriptionType subscriptionType; private String subscriptionName; private SubscriptionInitialPosition subscriptionPosition; - // Whether the subscriptions the functions created/used should be deleted when the functions is deleted + // Whether call consumer.seek(latest) to skip contents between last ask message and the latest message + private Boolean skipToLatest; private Integer maxMessageRetries = -1; private String deadLetterTopic; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java index 3e60111ddbe4b..426723804cad1 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.api.Record; @@ -73,6 +74,11 @@ public void open(Map config, SourceContext sourceContext) throws ConsumerBuilder cb = createConsumeBuilder(topic, pulsarSourceConsumerConfig); consumer = cb.subscribeAsync().join(); + + if (this.pulsarSourceConfig.getSkipToLatest() != null && this.pulsarSourceConfig.getSkipToLatest()) { + consumer.seek(MessageId.latest); + } + inputConsumers.add(consumer); } diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index e67899abd223e..101d45bc59cd7 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -165,6 +165,7 @@ message SourceSpec { bool cleanupSubscription = 11; SubscriptionPosition subscriptionPosition = 12; uint64 negativeAckRedeliveryDelayMs = 13; + bool skipToLatest = 14; } message SinkSpec { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 647210de33acf..d02fe5f788b5a 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -198,6 +198,12 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu sourceSpecBuilder.setSubscriptionPosition(subPosition); } + if (functionConfig.getSkipToLatest() != null) { + sourceSpecBuilder.setSkipToLatest(functionConfig.getSkipToLatest()); + } else { + sourceSpecBuilder.setSkipToLatest(false); + } + if (extractedDetails.getTypeArg0() != null) { sourceSpecBuilder.setTypeClassName(extractedDetails.getTypeArg0()); } else if (StringUtils.isNotEmpty(functionConfig.getInputTypeClassName())) { From 8cc979de411f9bca05ea95b16807f1860b79382e Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Fri, 24 Feb 2023 11:03:37 +0800 Subject: [PATCH 014/404] [improve][fn] Support e2e cryption in python instance (#18738) --- .../src/main/python/python_instance.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index f77ef38f76e4d..ac723232f4839 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -22,6 +22,7 @@ """python_instance.py: Python Instance for running python functions """ import base64 +import json import os import signal import time @@ -188,13 +189,16 @@ def run(self): consumer_conf.schemaProperties) Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) + crypto_key_reader = self.get_crypto_reader(consumer_conf.cryptoSpec) + consumer_args = { "consumer_type": mode, "schema": self.input_schema[topic], "message_listener": partial(self.message_listener, self.input_serdes[topic], self.input_schema[topic]), "unacked_messages_timeout_ms": int(self.timeout_ms) if self.timeout_ms else None, "initial_position": position, - "properties": properties + "properties": properties, + "crypto_key_reader": crypto_key_reader } if consumer_conf.HasField("receiverQueueSize"): consumer_args["receiver_queue_size"] = consumer_conf.receiverQueueSize.value @@ -343,6 +347,10 @@ def setup_producer(self): self.output_schema = self.get_schema(self.instance_config.function_details.sink.schemaType, self.instance_config.function_details.sink.typeClassName, self.instance_config.function_details.sink.schemaProperties) + crypto_key_reader = self.get_crypto_reader(self.instance_config.function_details.sink.producerSpec.cryptoSpec) + encryption_key = None + if crypto_key_reader is not None: + encryption_key = self.instance_config.function_details.sink.producerSpec.cryptoSpec.producerEncryptionKeyName[0] self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), @@ -355,6 +363,9 @@ def setup_producer(self): # set send timeout to be infinity to prevent potential deadlock with consumer # that might happen when consumer is blocked due to unacked messages send_timeout_millis=0, + # python client only supports one key for encryption + encryption_key=encryption_key, + crypto_key_reader=crypto_key_reader, properties=util.get_properties(util.getFullyQualifiedFunctionName( self.instance_config.function_details.tenant, self.instance_config.function_details.namespace, @@ -484,7 +495,6 @@ def close(self): if self.pulsar_client: self.pulsar_client.close() - # TODO: support other schemas: PROTOBUF, PROTOBUF_NATIVE, and KeyValue def get_schema(self, schema_type, type_class_name, schema_properties): schema = DEFAULT_SCHEMA @@ -526,4 +536,16 @@ def get_record_class(self, class_name): record_kclass = util.import_class(os.path.dirname(self.user_code), class_name) except: pass - return record_kclass \ No newline at end of file + return record_kclass + def get_crypto_reader(self, crypto_spec): + crypto_key_reader = None + if crypto_spec is not None: + try: + crypto_config = json.loads(crypto_spec.cryptoKeyReaderConfig) + if crypto_spec.cryptoKeyReaderClassName == "" or crypto_spec.cryptoKeyReaderClassName is None: + crypto_key_reader = pulsar.CryptoKeyReader(**crypto_config) + else: + crypto_key_reader = util.import_class(os.path.dirname(self.user_code), crypto_spec.cryptoKeyReaderClassName)(**crypto_config) + except Exception as e: + Log.error("Failed to load the crypto key reader from spec: %s, error: %s" % (crypto_spec, e)) + return crypto_key_reader From a26240bd0ce2cc1f77a7612e05cc142785766717 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Feb 2023 15:07:53 +0800 Subject: [PATCH 015/404] [fix][proxy] Fix JKS TLS transport (#19485) Signed-off-by: Zixuan Liu --- .../NettySSLContextAutoRefreshBuilder.java | 18 ++- .../proxy/server/DirectProxyHandler.java | 6 +- .../server/ProxyKeyStoreTlsTransportTest.java | 131 ++++++++++++++++++ 3 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java index 7b47eb9b8bb64..6d0cfb108bd0e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java @@ -47,7 +47,6 @@ public class NettySSLContextAutoRefreshBuilder extends SslContextAutoRefreshBuil protected String tlsKeyStorePassword; protected FileModifiedTimeUpdater tlsKeyStore; - protected AuthenticationDataProvider authData; protected final boolean isServer; // for server @@ -101,8 +100,14 @@ public NettySSLContextAutoRefreshBuilder(String sslProviderString, this.tlsAllowInsecureConnection = allowInsecureConnection; this.tlsProvider = sslProviderString; - this.authData = authData; - + if (authData != null) { + KeyStoreParams authParams = authData.getTlsKeyStoreParams(); + if (authParams != null) { + keyStoreTypeString = authParams.getKeyStoreType(); + keyStore = authParams.getKeyStorePath(); + keyStorePassword = authParams.getKeyStorePassword(); + } + } this.tlsKeyStoreType = keyStoreTypeString; this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); this.tlsKeyStorePassword = keyStorePassword; @@ -126,11 +131,10 @@ public synchronized KeyStoreSSLContext update() throws GeneralSecurityException, tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, tlsRequireTrustedClientCertOnConnect, tlsCiphers, tlsProtocols); } else { - KeyStoreParams authParams = authData.getTlsKeyStoreParams(); this.keyStoreSSLContext = KeyStoreSSLContext.createClientKeyStoreSslContext(tlsProvider, - authParams != null ? authParams.getKeyStoreType() : tlsKeyStoreType, - authParams != null ? authParams.getKeyStorePath() : tlsKeyStore.getFileName(), - authParams != null ? authParams.getKeyStorePassword() : tlsKeyStorePassword, + tlsKeyStoreType, + tlsKeyStore.getFileName(), + tlsKeyStorePassword, tlsAllowInsecureConnection, tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, tlsCiphers, tlsProtocols); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 1e9fd676573fd..23c7faa2d4bb7 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -127,9 +127,9 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) config.getBrokerClientTlsTrustStoreType(), config.getBrokerClientTlsTrustStore(), config.getBrokerClientTlsTrustStorePassword(), - null, - null, - null, + config.getBrokerClientTlsKeyStoreType(), + config.getBrokerClientTlsKeyStore(), + config.getBrokerClientTlsKeyStorePassword(), config.getBrokerClientTlsCiphers(), config.getBrokerClientTlsProtocols(), config.getTlsCertRefreshCheckDurationSec(), diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java new file mode 100644 index 0000000000000..5c4e40ed65a70 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static org.mockito.Mockito.doReturn; +import java.util.Optional; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "proxy") +public class ProxyKeyStoreTlsTransportTest extends MockedPulsarServiceBaseTest { + private ProxyService proxyService; + private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + + @Override + @BeforeMethod + protected void setup() throws Exception { + + // broker with JKS + conf.setWebServicePortTls(Optional.of(0)); + conf.setBrokerServicePortTls(Optional.of(0)); + conf.setTlsEnabledWithKeyStore(true); + conf.setTlsKeyStoreType(KEYSTORE_TYPE); + conf.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + conf.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + conf.setTlsTrustStoreType(KEYSTORE_TYPE); + conf.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + conf.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + conf.setTlsRequireTrustedClientCertOnConnect(true); + + internalSetup(); + + // proxy with JKS + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setServicePortTls(Optional.of(0)); + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setWebServicePortTls(Optional.of(0)); + proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setTlsEnabledWithKeyStore(true); + + proxyConfig.setTlsKeyStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + proxyConfig.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + proxyConfig.setTlsTrustStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + proxyConfig.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + + proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); + proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + + proxyConfig.setTlsRequireTrustedClientCertOnConnect(false); + + proxyConfig.setBrokerClientTlsEnabledWithKeyStore(true); + proxyConfig.setBrokerClientTlsKeyStore(CLIENT_KEYSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsKeyStorePassword(CLIENT_KEYSTORE_PW); + proxyConfig.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + + proxyService = Mockito.spy(new ProxyService(proxyConfig, + new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)))); + doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); + doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + + proxyService.start(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + internalCleanup(); + + proxyService.close(); + } + + protected PulsarClient newClient() throws Exception { + ClientBuilder clientBuilder = PulsarClient.builder() + .serviceUrl(proxyService.getServiceUrlTls()) + .useKeyStoreTls(true) + .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH) + .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW) + .tlsKeyStorePath(CLIENT_KEYSTORE_FILE_PATH) + .tlsKeyStorePassword(CLIENT_KEYSTORE_PW) + .allowTlsInsecureConnection(false); + return clientBuilder.build(); + } + + @Test + public void testProducer() throws Exception { + @Cleanup + PulsarClient client = newClient(); + @Cleanup + Producer producer = client.newProducer(Schema.BYTES) + .topic("persistent://sample/test/local/topic" + System.currentTimeMillis()) + .create(); + + for (int i = 0; i < 10; i++) { + producer.send("test".getBytes()); + } + } +} From 6d3e483bab0f960b21cb521fb3908eccd55993b6 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Feb 2023 17:09:20 +0800 Subject: [PATCH 016/404] [fix][client] Fix load the trust store file (#19483) Signed-off-by: Zixuan Liu --- .../util/keystoretls/KeyStoreSSLContext.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java index 4ed1826cfe9b1..c717127d085db 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java @@ -34,6 +34,7 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -150,25 +151,30 @@ public SSLContext createSSLContext() throws GeneralSecurityException, IOExceptio } // trust store - TrustManagerFactory trustManagerFactory; + TrustManagerFactory trustManagerFactory = null; if (this.allowInsecureConnection) { trustManagerFactory = InsecureTrustManagerFactory.INSTANCE; } else { - trustManagerFactory = provider != null - ? TrustManagerFactory.getInstance(tmfAlgorithm, provider) - : TrustManagerFactory.getInstance(tmfAlgorithm); - KeyStore trustStore = KeyStore.getInstance(trustStoreTypeString); - char[] passwordChars = trustStorePassword.toCharArray(); - try (FileInputStream inputStream = new FileInputStream(trustStorePath)) { - trustStore.load(inputStream, passwordChars); + if (!Strings.isNullOrEmpty(trustStorePath)) { + trustManagerFactory = provider != null + ? TrustManagerFactory.getInstance(tmfAlgorithm, provider) + : TrustManagerFactory.getInstance(tmfAlgorithm); + KeyStore trustStore = KeyStore.getInstance(trustStoreTypeString); + char[] passwordChars = trustStorePassword.toCharArray(); + try (FileInputStream inputStream = new FileInputStream(trustStorePath)) { + trustStore.load(inputStream, passwordChars); + } + trustManagerFactory.init(trustStore); } - trustManagerFactory.init(trustStore); + } + + TrustManager[] trustManagers = null; + if (trustManagerFactory != null) { + trustManagers = SecurityUtility.processConscryptTrustManagers(trustManagerFactory.getTrustManagers()); } // init - sslContext.init(keyManagers, SecurityUtility - .processConscryptTrustManagers(trustManagerFactory.getTrustManagers()), - new SecureRandom()); + sslContext.init(keyManagers, trustManagers, new SecureRandom()); this.sslContext = sslContext; return sslContext; } From 9e9ee02fe1d0e9434b5510c555a5906c9b427bdd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 24 Feb 2023 11:30:47 -0600 Subject: [PATCH 017/404] [improve][broker] Store nonBlank clientVersions that have spaces (#19616) Relates to: https://github.com/apache/pulsar/pull/19540 ### Motivation We currently filter out the `clientVersion` when it has a `" "` in it. As a consequence, we filter out the go client's version because of how it is made: https://github.com/apache/pulsar-client-go/blob/dedbdc45c63b06e6a12356785418cd906d6bab3c/pulsar/internal/version.go#L43 Given that we do not have any documented restrictions on the `clientVersion`, I think we should not drop clients that have a space in their name. I propose that we remove the filter logic to store all "valid" versions supplied by the client. ### Modifications * Update `ServerCnx` to store `clientVersion` when it has a space in its name. ### Verifying this change A new test is added. ### Does this pull request potentially affect one of the following parts: This is a backwards compatible change. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: skipping for this minor change that shouldn't make any tests fail --- .../pulsar/broker/service/ServerCnx.java | 2 +- .../pulsar/broker/service/ServerCnxTest.java | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 242947f6a0fd6..2d6bed3eb7ee3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -690,7 +690,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { log.debug("[{}] connect state change to : [{}]", remoteAddress, State.Connected.name()); } setRemoteEndpointProtocolVersion(clientProtoVersion); - if (isNotBlank(clientVersion) && !clientVersion.contains(" ") /* ignore default version: pulsar client */) { + if (isNotBlank(clientVersion)) { this.clientVersion = clientVersion.intern(); } if (brokerInterceptor != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index a29d6dac72023..4580f028de2b0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -143,6 +143,7 @@ import org.mockito.stubbing.Answer; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @SuppressWarnings("unchecked") @@ -293,6 +294,34 @@ public void testConnectCommandWithProtocolVersion() throws Exception { channel.finish(); } + @DataProvider(name = "clientVersions") + public Object[][] clientVersions() { + return new Object[][]{ + {"Pulsar Client", true}, + {"Pulsar Go 0.2.1", true}, + {"Pulsar-Client-Java-v1.15.2", true}, + {"pulsar-java-3.0.0", true}, + {"", false}, + {" ", false} + }; + } + + @Test(dataProvider = "clientVersions") + public void testStoreClientVersionWhenNotBlank(String clientVersion, boolean expectSetToClientVersion) throws Exception { + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + ByteBuf clientCommand = Commands.newConnect("", "", clientVersion); + channel.writeInbound(clientCommand); + + assertEquals(serverCnx.getState(), State.Connected); + assertTrue(getResponse() instanceof CommandConnected); + + assertEquals(serverCnx.getClientVersion(), expectSetToClientVersion ? clientVersion : null); + channel.finish(); + } + @Test(timeOut = 30000) public void testKeepAlive() throws Exception { resetChannel(); From de43ad0340f9fa857394449b7bfdbc4a339dcc92 Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Sat, 25 Feb 2023 01:55:52 +0800 Subject: [PATCH 018/404] [improve][broker] Print stack trace when got excpetion in ServerCnx (#19593) ### Motivation I meet a issue that NPE happend in ServerCnx.handleSend() but the debug log does not print stack trace. ![servercnx](https://user-images.githubusercontent.com/65590138/220540592-77fd107a-99c4-4eb3-ae84-6e6ceba96dfd.png) ### Modifications print stack trace when got excpetion in ServerCnx ### Verifying this change - [ ] Make sure that the change passes the CI checks. This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [ ] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 2d6bed3eb7ee3..b0256cddf6351 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -397,7 +397,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E // At default info level, suppress all subsequent exceptions that are thrown when the connection has already // failed if (log.isDebugEnabled()) { - log.debug("[{}] Got exception: {}", remoteAddress, cause); + log.debug("[{}] Got exception {}", remoteAddress, + ClientCnx.isKnownException(cause) ? cause : ExceptionUtils.getStackTrace(cause)); } } ctx.close(); From 939d065c9daeb5dced020ad838751b817dee022c Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Sun, 26 Feb 2023 11:56:13 +0800 Subject: [PATCH 019/404] [fix][broker] Remove useless load balancer items about MemoryResourceWeight (#19559) Motivation Even for a Broker with a very low load, its memory will grow slowly until GC is triggered. If memory is used as a load calculation item, the Bundle will be unloaded by mistake Modifications Remove memory as load calculation item --- conf/broker.conf | 9 +++++---- .../org/apache/pulsar/broker/ServiceConfiguration.java | 4 +++- .../loadbalance/impl/LeastResourceUsageWithWeight.java | 1 - .../pulsar/broker/loadbalance/impl/ThresholdShedder.java | 2 +- .../policies/data/loadbalancer/LocalBrokerData.java | 9 ++++++++- .../policies/data/loadbalancer/LocalBrokerDataTest.java | 8 +++----- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index f64d08a1de88c..3183c83a837a8 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1355,10 +1355,6 @@ loadBalancerBandwithOutResourceWeight=1.0 # It only takes effect in the ThresholdShedder strategy. loadBalancerCPUResourceWeight=1.0 -# The heap memory usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerMemoryResourceWeight=1.0 - # The direct memory usage weight when calculating new resource usage. # It only takes effect in the ThresholdShedder strategy. loadBalancerDirectMemoryResourceWeight=1.0 @@ -1669,6 +1665,11 @@ strictBookieAffinityEnabled=false # These settings are left here for compatibility +# The heap memory usage weight when calculating new resource usage. +# It only takes effect in the ThresholdShedder strategy. +# Deprecated: Memory is no longer used as a load balancing item +loadBalancerMemoryResourceWeight=1.0 + # Zookeeper quorum connection string # Deprecated: use metadataStoreUrl instead zookeeperServers= diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 4f2c8e72e131d..5a85be11745e2 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2341,10 +2341,12 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private double loadBalancerCPUResourceWeight = 1.0; + @Deprecated(since = "3.0.0") @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "Memory Resource Usage Weight" + doc = "Memory Resource Usage Weight. Deprecated: Memory is no longer used as a load balancing item.", + deprecated = true ) private double loadBalancerMemoryResourceWeight = 1.0; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java index 35dd7282101b5..f50f9ed36538a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java @@ -98,7 +98,6 @@ private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData } double resourceUsage = brokerData.getLocalData().getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), - conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), conf.getLoadBalancerBandwithInResourceWeight(), conf.getLoadBalancerBandwithOutResourceWeight()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index c75c9c8a3178c..e2f1a7808fe91 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -168,7 +168,7 @@ private double updateAvgResourceUsage(String broker, LocalBrokerData localBroker brokerAvgResourceUsage.get(broker); double resourceUsage = localBrokerData.getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), - conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), + conf.getLoadBalancerDirectMemoryResourceWeight(), conf.getLoadBalancerBandwithInResourceWeight(), conf.getLoadBalancerBandwithOutResourceWeight()); historyUsage = historyUsage == null diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java index 030ecc63be545..8c0c008e0a555 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java @@ -249,7 +249,7 @@ public String printResourceUsage() { cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), bandwidthOut.percentUsage()); } - + @Deprecated public double getMaxResourceUsageWithWeight(final double cpuWeight, final double memoryWeight, final double directMemoryWeight, final double bandwidthInWeight, final double bandwidthOutWeight) { @@ -257,6 +257,13 @@ public double getMaxResourceUsageWithWeight(final double cpuWeight, final double directMemory.percentUsage() * directMemoryWeight, bandwidthIn.percentUsage() * bandwidthInWeight, bandwidthOut.percentUsage() * bandwidthOutWeight) / 100; } + public double getMaxResourceUsageWithWeight(final double cpuWeight, + final double directMemoryWeight, final double bandwidthInWeight, + final double bandwidthOutWeight) { + return max(cpu.percentUsage() * cpuWeight, + directMemory.percentUsage() * directMemoryWeight, bandwidthIn.percentUsage() * bandwidthInWeight, + bandwidthOut.percentUsage() * bandwidthOutWeight) / 100; + } public static double max(double... args) { double max = Double.NEGATIVE_INFINITY; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java index a95f019234caa..db55ecfe5035a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java @@ -43,7 +43,6 @@ public void testLocalBrokerDataDeserialization() { public void testMaxResourceUsage() { LocalBrokerData data = new LocalBrokerData(); data.setCpu(new ResourceUsage(1.0, 100.0)); - data.setMemory(new ResourceUsage(800.0, 200.0)); data.setDirectMemory(new ResourceUsage(2.0, 100.0)); data.setBandwidthIn(new ResourceUsage(3.0, 100.0)); data.setBandwidthOut(new ResourceUsage(4.0, 100.0)); @@ -51,11 +50,10 @@ public void testMaxResourceUsage() { double epsilon = 0.00001; double weight = 0.5; // skips memory usage - assertEquals(data.getMaxResourceUsage(), 0.04, epsilon); + assertEquals(data.getMaxResourceUsage(), data.getBandwidthOut().percentUsage() / 100, epsilon); - assertEquals( - data.getMaxResourceUsageWithWeight( - weight, weight, weight, weight, weight), 2.0, epsilon); + assertEquals(data.getMaxResourceUsageWithWeight(weight, weight, weight, weight), + data.getBandwidthOut().percentUsage() * weight / 100, epsilon); } /* From 5d6932137d76d544f939bef27df25f61b4a4d00d Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 27 Feb 2023 08:56:47 +0800 Subject: [PATCH 020/404] [feat] PIP-242 part 1 Introduce configuration `strictTopicNameEnabled` (#19582) --- .../pulsar/broker/ServiceConfiguration.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 5a85be11745e2..c18367f265506 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1399,6 +1399,31 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se doc = "Enable or disable system topic.") private boolean systemTopicEnabled = true; + @FieldContext( + category = CATEGORY_SERVER, + doc = "# Enable strict topic name check. Which includes two parts as follows:\n" + + "# 1. Mark `-partition-` as a keyword.\n" + + "# E.g.\n" + + " Create a non-partitioned topic.\n" + + " No corresponding partitioned topic\n" + + " - persistent://public/default/local-name (passed)\n" + + " - persistent://public/default/local-name-partition-z (rejected by keyword)\n" + + " - persistent://public/default/local-name-partition-0 (rejected by keyword)\n" + + " Has corresponding partitioned topic, partitions=2 and topic partition name " + + "is persistent://public/default/local-name\n" + + " - persistent://public/default/local-name-partition-0 (passed," + + " Because it is the partition topic's sub-partition)\n" + + " - persistent://public/default/local-name-partition-z (rejected by keyword)\n" + + " - persistent://public/default/local-name-partition-4 (rejected," + + " Because it exceeds the number of maximum partitions)\n" + + " Create a partitioned topic(topic metadata)\n" + + " - persistent://public/default/local-name (passed)\n" + + " - persistent://public/default/local-name-partition-z (rejected by keyword)\n" + + " - persistent://public/default/local-name-partition-0 (rejected by keyword)\n" + + "# 2. Allowed alphanumeric (a-zA-Z_0-9) and these special chars -=:. for topic name.\n" + + "# NOTE: This flag will be removed in some major releases in the future.\n") + private boolean strictTopicNameEnabled = false; + @FieldContext( category = CATEGORY_SCHEMA, doc = "The schema compatibility strategy to use for system topics" From dc02c404223db757702a8e1318e970adc37a660b Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Mon, 27 Feb 2023 16:23:41 +0800 Subject: [PATCH 021/404] [fix][doc] update link for Pulsar PR Naming Convention Guide (#19642) --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9dbe56a13717b..01ac26570b2d1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,7 @@ 4.1.12.1 5.1.0 - 4.1.87.Final - 0.0.17.Final + 4.1.89.Final + 0.0.18.Final 9.4.48.v20220622 2.5.2 2.34 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d11ab20397db7..64c488c38bbe9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -231,21 +231,21 @@ The Apache Software License, Version 2.0 - commons-compress-1.21.jar - commons-lang3-3.11.jar * Netty - - netty-buffer-4.1.87.Final.jar - - netty-codec-4.1.87.Final.jar - - netty-codec-dns-4.1.87.Final.jar - - netty-codec-http-4.1.87.Final.jar - - netty-codec-haproxy-4.1.87.Final.jar - - netty-codec-socks-4.1.87.Final.jar - - netty-handler-proxy-4.1.87.Final.jar - - netty-common-4.1.87.Final.jar - - netty-handler-4.1.87.Final.jar + - netty-buffer-4.1.89.Final.jar + - netty-codec-4.1.89.Final.jar + - netty-codec-dns-4.1.89.Final.jar + - netty-codec-http-4.1.89.Final.jar + - netty-codec-haproxy-4.1.89.Final.jar + - netty-codec-socks-4.1.89.Final.jar + - netty-handler-proxy-4.1.89.Final.jar + - netty-common-4.1.89.Final.jar + - netty-handler-4.1.89.Final.jar - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.87.Final.jar - - netty-resolver-dns-4.1.87.Final.jar - - netty-resolver-dns-classes-macos-4.1.87.Final.jar - - netty-resolver-dns-native-macos-4.1.87.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.87.Final-osx-x86_64.jar + - netty-resolver-4.1.89.Final.jar + - netty-resolver-dns-4.1.89.Final.jar + - netty-resolver-dns-classes-macos-4.1.89.Final.jar + - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.56.Final.jar - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar @@ -253,15 +253,15 @@ The Apache Software License, Version 2.0 - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - netty-tcnative-classes-2.0.56.Final.jar - - netty-transport-4.1.87.Final.jar - - netty-transport-classes-epoll-4.1.87.Final.jar - - netty-transport-native-epoll-4.1.87.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.87.Final.jar - - netty-transport-native-unix-common-4.1.87.Final-linux-x86_64.jar - - netty-codec-http2-4.1.87.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.17.Final.jar - - netty-incubator-transport-native-io_uring-0.0.17.Final-linux-x86_64.jar - - netty-incubator-transport-native-io_uring-0.0.17.Final-linux-aarch_64.jar + - netty-transport-4.1.89.Final.jar + - netty-transport-classes-epoll-4.1.89.Final.jar + - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.89.Final.jar + - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar + - netty-codec-http2-4.1.89.Final.jar + - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar + - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar + - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar * GRPC - grpc-api-1.45.1.jar - grpc-context-1.45.1.jar From 69fb3c2ca3faa32ff12fd1270730b3517ea69220 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 28 Feb 2023 12:17:33 +0100 Subject: [PATCH 028/404] [fix][client] Fix race condition that leads to caching failed CompletableFutures in ConnectionPool (#19661) --- .../apache/pulsar/client/impl/ConnectionPool.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 3a9a2b9b7ab94..1420d81c688ee 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -216,6 +216,15 @@ public CompletableFuture getConnection(InetSocketAddress logicalAddre pool.computeIfAbsent(logicalAddress, a -> new ConcurrentHashMap<>()); CompletableFuture completableFuture = innerPool .computeIfAbsent(randomKey, k -> createConnection(logicalAddress, physicalAddress, randomKey)); + if (completableFuture.isCompletedExceptionally()) { + // we cannot cache a failed connection, so we remove it from the pool + // there is a race condition in which + // cleanupConnection is called before caching this result + // and so the clean up fails + cleanupConnection(logicalAddress, randomKey, completableFuture); + return completableFuture; + } + return completableFuture.thenCompose(clientCnx -> { // If connection already release, create a new one. if (clientCnx.getIdleState().isReleased()) { @@ -274,6 +283,10 @@ private CompletableFuture createConnection(InetSocketAddress logicalA }).exceptionally(exception -> { log.warn("[{}] Connection handshake failed: {}", cnx.channel(), exception.getMessage()); cnxFuture.completeExceptionally(exception); + // this cleanupConnection may happen before that the + // CompletableFuture is cached into the "pool" map, + // it is not enough to clean it here, we need to clean it + // in the "pool" map when the CompletableFuture is cached cleanupConnection(logicalAddress, connectionKey, cnxFuture); cnx.ctx().close(); return null; From 7dad5791bddd25bbfc4b154056ac723bb5d64ede Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 1 Mar 2023 09:13:04 +0800 Subject: [PATCH 029/404] [fix][broker] Fix BucketDelayedDeliveryTracker merge issues (#19615) --- .../BucketDelayedDeliveryTrackerFactory.java | 3 +- .../bucket/BucketDelayedDeliveryTracker.java | 64 ++++++++++++++----- .../delayed/bucket/ImmutableBucket.java | 22 +++++-- .../BucketDelayedDeliveryTrackerTest.java | 9 +++ .../persistent/BucketDelayedDeliveryTest.java | 5 ++ 5 files changed, 80 insertions(+), 23 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java index 16648d84e9fba..ae9cb23ceb922 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java @@ -49,6 +49,7 @@ public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrack public void initialize(PulsarService pulsarService) throws Exception { ServiceConfiguration config = pulsarService.getConfig(); bucketSnapshotStorage = new BookkeeperBucketSnapshotStorage(pulsarService); + bucketSnapshotStorage.start(); this.timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-delayed-delivery"), config.getDelayedDeliveryTickTimeMillis(), TimeUnit.MILLISECONDS); this.tickTimeMillis = config.getDelayedDeliveryTickTimeMillis(); @@ -63,7 +64,7 @@ public void initialize(PulsarService pulsarService) throws Exception { public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { return new BucketDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, isDelayedDeliveryDeliverAtTimeStrict, bucketSnapshotStorage, delayedDeliveryMinIndexCountPerBucket, - delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds, + TimeUnit.SECONDS.toMillis(delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds), delayedDeliveryMaxNumBuckets); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 77c1dfb1eea14..bd4ef92cc7320 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -55,6 +55,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; +import org.roaringbitmap.RoaringBitmap; @Slf4j @ThreadSafe @@ -64,7 +65,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final long minIndexCountPerBucket; - private final long timeStepPerBucketSnapshotSegment; + private final long timeStepPerBucketSnapshotSegmentInMillis; private final int maxNumBuckets; @@ -84,21 +85,21 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, int maxNumBuckets) { this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, - bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); + bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis, maxNumBuckets); } public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, int maxNumBuckets) { super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.minIndexCountPerBucket = minIndexCountPerBucket; - this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; + this.timeStepPerBucketSnapshotSegmentInMillis = timeStepPerBucketSnapshotSegmentInMillis; this.maxNumBuckets = maxNumBuckets; this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); @@ -255,7 +256,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver && lastMutableBucket.size() >= minIndexCountPerBucket && !lastMutableBucket.isEmpty()) { Pair immutableBucketDelayedIndexPair = - lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegment, + lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegmentInMillis, this.sharedBucketPriorityQueue); afterCreateImmutableBucket(immutableBucketDelayedIndexPair); lastMutableBucket.resetLastMutableBucketRange(); @@ -303,17 +304,21 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; if (numberMessages < minNumberMessages) { minNumberMessages = (int) numberMessages; - minIndex = i; + if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()) { + minIndex = i; + } } } + + if (minIndex == -1) { + log.warn("[{}] Can't find able merged bucket", dispatcher.getName()); + return CompletableFuture.completedFuture(null); + } return asyncMergeBucketSnapshot(values.get(minIndex), values.get(minIndex + 1)); } private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA, ImmutableBucket bucketB) { - immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); - CompletableFuture snapshotCreateFutureA = bucketA.getSnapshotCreateFuture().orElse(CompletableFuture.completedFuture(null)); CompletableFuture snapshotCreateFutureB = @@ -328,16 +333,41 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB .thenAccept(combinedDelayedIndexQueue -> { Pair immutableBucketDelayedIndexPair = lastMutableBucket.createImmutableBucketAndAsyncPersistent( - timeStepPerBucketSnapshotSegment, sharedBucketPriorityQueue, + timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId); + + // Merge bit map to new bucket + Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); + Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); + Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); + delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { + delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { + if (bitMapA == null) { + return bitMapB; + } + + bitMapA.or(bitMapB); + return bitMapA; + }); + }); + immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); + afterCreateImmutableBucket(immutableBucketDelayedIndexPair); - immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() - .orElse(CompletableFuture.completedFuture(null)).thenCompose(___ -> { - CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); - CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); - return CompletableFuture.allOf(removeAFuture, removeBFuture); - }); + CompletableFuture snapshotCreateFuture = CompletableFuture.completedFuture(null); + if (immutableBucketDelayedIndexPair != null) { + snapshotCreateFuture = immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() + .orElse(CompletableFuture.completedFuture(null)); + } + + snapshotCreateFuture.thenCompose(___ -> { + CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); + CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); + return CompletableFuture.allOf(removeAFuture, removeBFuture); + }); + + immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); + immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); }); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 8348b4999ed80..3e9c577454fed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -24,9 +24,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -134,7 +132,11 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, } CompletableFuture> getRemainSnapshotSegment() { - return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), currentSegmentEntryId, + int nextSegmentEntryId = currentSegmentEntryId + 1; + if (nextSegmentEntryId > lastSegmentEntryId) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } + return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, lastSegmentEntryId); } @@ -155,9 +157,19 @@ void clear(boolean delete) { getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> { if (delete) { snapshotGenerateFuture.cancel(true); + String bucketKey = bucketKey(); + long bucketId = getAndUpdateBucketId(); try { - asyncDeleteBucketSnapshot().get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { + // Because bucketSnapshotStorage.deleteBucketSnapshot may be use the same thread with clear, + // so we can't block deleteBucketSnapshot when clearing the bucket snapshot. + removeBucketCursorProperty(bucketKey()) + .thenApply(__ -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).exceptionally(ex -> { + log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", + bucketId, bucketKey, ex); + return null; + })).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", bucketId, bucketKey, e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 0a2a76ec339eb..920f2cf2b64b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.List; import java.util.NavigableMap; +import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; @@ -253,5 +254,13 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { int size = tracker.getImmutableBuckets().asMapOfRanges().size(); assertEquals(10, size); + + clockTime.set(110 * 10); + + NavigableSet scheduledMessages = tracker.getScheduledMessages(110); + for (int i = 1; i <= 110; i++) { + PositionImpl position = scheduledMessages.pollFirst(); + assertEquals(position, PositionImpl.get(i, i)); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index a54c0ce794cf0..5d81ba8bc0261 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -30,6 +30,11 @@ public class BucketDelayedDeliveryTest extends DelayedDeliveryTest { @Override public void setup() throws Exception { conf.setDelayedDeliveryTrackerFactoryClassName(BucketDelayedDeliveryTrackerFactory.class.getName()); + conf.setDelayedDeliveryMaxNumBuckets(10); + conf.setDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds(1); + conf.setDelayedDeliveryMinIndexCountPerBucket(50); + conf.setManagedLedgerMaxEntriesPerLedger(50); + conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0); super.setup(); } From 145e985f7b7ef981d33036b79b24fd5a4e27d43c Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Wed, 1 Mar 2023 09:43:13 +0800 Subject: [PATCH 030/404] [improve][broker] Replace ScheduledExecutorService to ExecutorService in ModularLoadManagerImpl (#19656) --- .../loadbalance/impl/ModularLoadManagerImpl.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 7a933908962ec..c1afe6007f6bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -33,9 +33,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -174,8 +174,8 @@ public class ModularLoadManagerImpl implements ModularLoadManager { // Pulsar service used to initialize this. private PulsarService pulsar; - // Executor service used to regularly update broker data. - private final ScheduledExecutorService scheduler; + // Executor service used to update broker data. + private final ExecutorService executors; // check if given broker can load persistent/non-persistent topic private final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate; @@ -215,7 +215,7 @@ public ModularLoadManagerImpl() { loadData = new LoadData(); loadSheddingPipeline = new ArrayList<>(); preallocatedBundleToBroker = new ConcurrentHashMap<>(); - scheduler = Executors.newSingleThreadScheduledExecutor( + executors = Executors.newSingleThreadExecutor( new ExecutorProvider.ExtendedThreadFactory("pulsar-modular-load-manager")); this.brokerToFailureDomainMap = new HashMap<>(); this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); @@ -276,7 +276,7 @@ public void initialize(final PulsarService pulsar) { // register listeners for domain changes pulsar.getPulsarResources().getClusterResources().getFailureDomainResources() .registerListener(__ -> { - scheduler.execute(() -> refreshBrokerToFailureDomainMap()); + executors.execute(() -> refreshBrokerToFailureDomainMap()); }); loadSheddingPipeline.add(createLoadSheddingStrategy()); @@ -290,7 +290,7 @@ public void handleDataNotification(Notification t) { }); try { - scheduler.execute(ModularLoadManagerImpl.this::updateAll); + executors.execute(ModularLoadManagerImpl.this::updateAll); } catch (RejectedExecutionException e) { // Executor is shutting down } @@ -982,7 +982,7 @@ public void start() throws PulsarServerException { */ @Override public void stop() throws PulsarServerException { - scheduler.shutdownNow(); + executors.shutdownNow(); try { brokersData.close(); From a12f5541cee0d51fad654e97487edf1140de3286 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 1 Mar 2023 00:58:33 -0800 Subject: [PATCH 031/404] [improve][broker] PIP-192: Added VersionId in ServiceUnitStateData (#19620) --- .../channel/ServiceUnitStateChannelImpl.java | 59 +++-- .../ServiceUnitStateCompactionStrategy.java | 11 +- .../channel/ServiceUnitStateData.java | 15 +- .../channel/ServiceUnitStateChannelTest.java | 12 +- ...erviceUnitStateCompactionStrategyTest.java | 241 +++++++++++------- .../channel/ServiceUnitStateDataTest.java | 20 +- .../ServiceUnitStateCompactionTest.java | 36 ++- 7 files changed, 251 insertions(+), 143 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 9f205f85c5454..4819a54fcad73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -103,7 +103,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { NamespaceName.SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec - + public static final long VERSION_ID_INIT = 1; // initial versionId private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately @@ -451,11 +451,25 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } } + private long getNextVersionId(String serviceUnit) { + var data = tableview.get(serviceUnit); + return getNextVersionId(data); + } + + private long getNextVersionId(ServiceUnitStateData data) { + return data == null ? VERSION_ID_INIT : data.versionId() + 1; + } + public CompletableFuture publishAssignEventAsync(String serviceUnit, String broker) { + if (!validateChannelState(Started, true)) { + return CompletableFuture.failedFuture( + new IllegalStateException("Invalid channel state:" + channelState.name())); + } EventType eventType = Assign; eventCounters.get(eventType).getTotal().incrementAndGet(); CompletableFuture getOwnerRequest = deferGetOwnerRequest(serviceUnit); - pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker)) + + pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker, getNextVersionId(serviceUnit))) .whenComplete((__, ex) -> { if (ex != null) { getOwnerRequests.remove(serviceUnit, getOwnerRequest); @@ -469,16 +483,20 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str } public CompletableFuture publishUnloadEventAsync(Unload unload) { + if (!validateChannelState(Started, true)) { + return CompletableFuture.failedFuture( + new IllegalStateException("Invalid channel state:" + channelState.name())); + } EventType eventType = Unload; eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = unload.serviceUnit(); CompletableFuture future; if (isTransferCommand(unload)) { future = pubAsync(serviceUnit, new ServiceUnitStateData( - Assigning, unload.destBroker().get(), unload.sourceBroker())); + Assigning, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit))); } else { future = pubAsync(serviceUnit, new ServiceUnitStateData( - Releasing, unload.sourceBroker())); + Releasing, unload.sourceBroker(), getNextVersionId(serviceUnit))); } return future.whenComplete((__, ex) -> { @@ -489,10 +507,15 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { } public CompletableFuture publishSplitEventAsync(Split split) { + if (!validateChannelState(Started, true)) { + return CompletableFuture.failedFuture( + new IllegalStateException("Invalid channel state:" + channelState.name())); + } EventType eventType = Split; eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = split.serviceUnit(); - ServiceUnitStateData next = new ServiceUnitStateData(Splitting, split.sourceBroker()); + ServiceUnitStateData next = + new ServiceUnitStateData(Splitting, split.sourceBroker(), getNextVersionId(serviceUnit)); return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); @@ -599,7 +622,8 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.broker())) { ServiceUnitStateData next = new ServiceUnitStateData( - isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker()); + isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker(), + getNextVersionId(data)); pubAsync(serviceUnit, next) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } @@ -608,7 +632,8 @@ private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { if (isTransferCommand(data)) { if (isTargetBroker(data.sourceBroker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker()); + ServiceUnitStateData next = + new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker(), getNextVersionId(data)); // TODO: when close, pass message to clients to connect to the new broker closeServiceUnit(serviceUnit) .thenCompose(__ -> pubAsync(serviceUnit, next)) @@ -616,7 +641,7 @@ private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { } } else { if (isTargetBroker(data.broker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker()); + ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker(), getNextVersionId(data)); closeServiceUnit(serviceUnit) .thenCompose(__ -> pubAsync(serviceUnit, next)) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); @@ -660,10 +685,6 @@ private void handleInitEvent(String serviceUnit) { } private CompletableFuture pubAsync(String serviceUnit, ServiceUnitStateData data) { - if (!validateChannelState(Started, true)) { - return CompletableFuture.failedFuture( - new IllegalStateException("Invalid channel state:" + channelState.name())); - } CompletableFuture future = new CompletableFuture<>(); producer.newMessage() .key(serviceUnit) @@ -774,7 +795,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); return; } - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker()); + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), VERSION_ID_INIT); NamespaceBundles targetNsBundle = splitBundlesPair.getLeft(); List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight()); List successPublishedBundles = @@ -812,7 +833,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.thenAccept(r -> { // Delete the old bundle - pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker())).thenRun(() -> { + pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker(), getNextVersionId(data))) + .thenRun(() -> { // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); // TODO: Update the load data immediately if needed. @@ -938,7 +960,7 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa Optional selectedBroker = brokerSelector.select(availableBrokers, null, getContext()); if (selectedBroker.isPresent()) { - var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true); + var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData)); log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", serviceUnit, orphanData, override); pubAsync(serviceUnit, override).whenComplete((__, e) -> { @@ -1007,19 +1029,20 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce private Optional getOverrideStateData(String serviceUnit, ServiceUnitStateData orphanData, Set availableBrokers, LoadManagerContext context) { + long nextVersionId = getNextVersionId(orphanData); if (isTransferCommand(orphanData)) { // rollback to the src - return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true)); + return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); } else if (orphanData.state() == Assigning) { // assign // roll-forward to another broker Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); if (selectedBroker.isEmpty()) { return Optional.empty(); } - return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true)); + return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); } else if (orphanData.state() == Splitting || orphanData.state() == Releasing) { // rollback to the target broker for split and unload - return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true)); + return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true, nextVersionId)); } else { var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", serviceUnit, orphanData); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index d2a585af9d9d5..8af0f0c027da4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -50,14 +50,19 @@ public void checkBrokers(boolean check) { public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { if (to == null) { return false; - } else if (to.force()) { - return false; } + // Skip the compaction case where from = null and to.versionId > 1 + if (from != null && from.versionId() + 1 != to.versionId()) { + return true; + } + + if (to.force()) { + return false; + } ServiceUnitState prevState = state(from); ServiceUnitState state = state(to); - if (!ServiceUnitState.isValidTransition(prevState, state)) { return true; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index 6a04431de64d5..ef25acff10a4b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; - import java.util.Objects; import org.apache.commons.lang3.StringUtils; @@ -28,7 +27,7 @@ */ public record ServiceUnitStateData( - ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp) { + ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp, long versionId) { public ServiceUnitStateData { Objects.requireNonNull(state); @@ -37,16 +36,16 @@ public record ServiceUnitStateData( } } - public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker) { - this(state, broker, sourceBroker, false, System.currentTimeMillis()); + public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker, long versionId) { + this(state, broker, sourceBroker, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker) { - this(state, broker, null, false, System.currentTimeMillis()); + public ServiceUnitStateData(ServiceUnitState state, String broker, long versionId) { + this(state, broker, null, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force) { - this(state, broker, null, force, System.currentTimeMillis()); + public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force, long versionId) { + this(state, broker, null, force, System.currentTimeMillis(), versionId); } public static ServiceUnitState state(ServiceUnitStateData data) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 49eee6ecb7aef..6aa8fe387605b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -924,26 +924,26 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx @Test(priority = 11) public void ownerLookupCountTests() throws IllegalAccessException { - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1", 1)); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1", 1)); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 1a4aba15f9e6f..0cd05d8bd7559 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -32,110 +32,173 @@ @Test(groups = "broker") public class ServiceUnitStateCompactionStrategyTest { ServiceUnitStateCompactionStrategy strategy = new ServiceUnitStateCompactionStrategy(); + String dst = "dst"; + String src = "src"; ServiceUnitStateData data(ServiceUnitState state) { - return new ServiceUnitStateData(state, "broker"); + return new ServiceUnitStateData(state, "broker", 1); } ServiceUnitStateData data(ServiceUnitState state, String dst) { - return new ServiceUnitStateData(state, dst, null); + return new ServiceUnitStateData(state, dst, null, 1); } + ServiceUnitStateData data(ServiceUnitState state, String src, String dst) { - return new ServiceUnitStateData(state, dst, src); + return new ServiceUnitStateData(state, dst, src, 1); + } + + ServiceUnitStateData data2(ServiceUnitState state) { + return new ServiceUnitStateData(state, "broker", 2); + } + + ServiceUnitStateData data2(ServiceUnitState state, String dst) { + return new ServiceUnitStateData(state, dst, null, 2); + } + + ServiceUnitStateData data2(ServiceUnitState state, String src, String dst) { + return new ServiceUnitStateData(state, dst, src, 2); } @Test - public void test() throws InterruptedException { - String dst = "dst"; - String src = "src"; + public void testVersionId(){ + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Assigning, dst, 1), + new ServiceUnitStateData(Assigning, dst, 1))); + + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Assigning, dst, 1), + new ServiceUnitStateData(Assigning, dst, 2))); + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, 10), + new ServiceUnitStateData(Assigning, "broker2", dst, 11))); + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE), + new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 1))); + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE + 1), + new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 2))); + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, 10), + new ServiceUnitStateData(Assigning, "broker2", dst, 5))); + + } + + @Test + public void testForce(){ assertFalse(strategy.shouldKeepLeft( - new ServiceUnitStateData(Init, dst), - new ServiceUnitStateData(Init, dst, true))); + new ServiceUnitStateData(Init, dst, 1), + new ServiceUnitStateData(Init, dst, true, 2))); + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Init, dst, 1), + new ServiceUnitStateData(Init, dst, true, 1))); + } + + @Test + public void testTombstone() { + assertFalse(strategy.shouldKeepLeft( + data(Init), null)); + assertFalse(strategy.shouldKeepLeft( + data(Assigning), null)); assertFalse(strategy.shouldKeepLeft( data(Owned), null)); + assertFalse(strategy.shouldKeepLeft( + data(Releasing), null)); + assertFalse(strategy.shouldKeepLeft( + data(Splitting), null)); + assertFalse(strategy.shouldKeepLeft( + data(Free), null)); + assertFalse(strategy.shouldKeepLeft( + data(Deleted), null)); + } + + @Test + public void testTransitionsAndBrokers() { + + assertTrue(strategy.shouldKeepLeft(data(Init), data2(Init))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Free))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Assigning))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Owned))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Releasing))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Splitting))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Deleted))); + + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data2(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, src, dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data2(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data2(Releasing, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Deleted, dst))); + + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, "dst1"))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Releasing, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Releasing, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Splitting, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Splitting, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Splitting, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Deleted, dst))); + + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Init))); + assertFalse(strategy.shouldKeepLeft(data(Releasing), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Free, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Free, "src2", dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Owned, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Owned, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data2(Owned, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Deleted, dst))); + + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, src, "dst1"), data2(Deleted, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "dst1"), data2(Deleted, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "src1", dst), data2(Deleted, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, dst), data2(Deleted, dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, src, dst), data2(Deleted, src, dst))); + + assertFalse(strategy.shouldKeepLeft(data(Deleted), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Deleted))); - assertTrue(strategy.shouldKeepLeft(data(Init), data(Init))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Free))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Assigning))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Owned))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Releasing))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Splitting))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Deleted))); - - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data(Owned, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, src, dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data(Releasing, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data(Releasing, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Splitting, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Deleted, dst))); - - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Assigning, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, "dst1"))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Releasing, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Releasing, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Releasing, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Splitting, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Splitting, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Splitting, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Splitting, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Deleted, dst))); - - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Init))); - assertFalse(strategy.shouldKeepLeft(data(Releasing), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Free, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Free, "src2", dst))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Owned, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data(Owned, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data(Owned, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Deleted, dst))); - - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Splitting, src, "dst1"), data(Deleted, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Splitting, "dst1"), data(Deleted, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Splitting, "src1", dst), data(Deleted, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Splitting, dst), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Splitting, src, dst), data(Deleted, src, dst))); - - assertFalse(strategy.shouldKeepLeft(data(Deleted), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Deleted))); - - assertFalse(strategy.shouldKeepLeft(data(Free), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Free))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Assigning, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Deleted))); + assertFalse(strategy.shouldKeepLeft(data(Free), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Free))); + assertFalse(strategy.shouldKeepLeft(data(Free), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Deleted))); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java index 9617c8a8c2bd0..a48e2a4db8b37 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java @@ -33,15 +33,16 @@ public class ServiceUnitStateDataTest { @Test public void testConstructors() throws InterruptedException { - ServiceUnitStateData data1 = new ServiceUnitStateData(Owned, "A"); + ServiceUnitStateData data1 = new ServiceUnitStateData(Owned, "A", 1); assertEquals(data1.state(), Owned); assertEquals(data1.broker(), "A"); assertNull(data1.sourceBroker()); - assertThat(data1.timestamp()).isGreaterThan(0);; + assertThat(data1.timestamp()).isGreaterThan(0); + ; Thread.sleep(10); - ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B"); + ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B", 1); assertEquals(data2.state(), Assigning); assertEquals(data2.broker(), "A"); assertEquals(data2.sourceBroker(), "B"); @@ -50,23 +51,28 @@ public void testConstructors() throws InterruptedException { @Test(expectedExceptions = NullPointerException.class) public void testNullState() { - new ServiceUnitStateData(null, "A"); + new ServiceUnitStateData(null, "A", 1); } @Test(expectedExceptions = IllegalArgumentException.class) public void testNullBroker() { - new ServiceUnitStateData(Owned, null); + new ServiceUnitStateData(Owned, null, 1); } @Test(expectedExceptions = IllegalArgumentException.class) public void testEmptyBroker() { - new ServiceUnitStateData(Owned, ""); + new ServiceUnitStateData(Owned, "", 1); + } + + @Test + public void testZeroVersionId() { + new ServiceUnitStateData(Owned, "A", Long.MAX_VALUE + 1); } @Test public void jsonWriteAndReadTest() throws JsonProcessingException { ObjectMapper mapper = ObjectMapperFactory.create(); - final ServiceUnitStateData src = new ServiceUnitStateData(Assigning, "A", "B"); + final ServiceUnitStateData src = new ServiceUnitStateData(Assigning, "A", "B", 1); String json = mapper.writeValueAsString(src); ServiceUnitStateData dst = mapper.readValue(json, ServiceUnitStateData.class); assertEquals(dst, src); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 4c1d4f7d2a89d..543b7c629ac5f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -54,6 +54,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -88,20 +89,24 @@ public class ServiceUnitStateCompactionTest extends MockedPulsarServiceBaseTest private ServiceUnitState testState = Init; + private ServiceUnitStateData testData = null; + private static Random RANDOM = new Random(); private ServiceUnitStateData testValue(ServiceUnitState state, String broker) { if (state == Init) { - return null; + testData = null; + } else { + testData = new ServiceUnitStateData(state, broker, versionId(testData) + 1); } - return new ServiceUnitStateData(state, broker); + + return testData; } private ServiceUnitStateData testValue(String broker) { - ServiceUnitState to = nextValidStateNonSplit(testState); - testState = to; - return testValue(to, broker); + testState = nextValidStateNonSplit(testState); + return testValue(testState, broker); } private ServiceUnitState nextValidState(ServiceUnitState from) { @@ -149,6 +154,7 @@ public void setup() throws Exception { strategy.checkBrokers(false); testState = Init; + testData = null; } @@ -202,13 +208,14 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException { ServiceUnitState state = invalid ? nextInvalidState(prevState) : nextValidState(prevState); ServiceUnitStateData value; + long versionId = versionId(prev) + 1; if (invalid) { - value = new ServiceUnitStateData(state, key + ":" + j, false); + value = new ServiceUnitStateData(state, key + ":" + j, false, versionId); } else { if (state == Init) { - value = new ServiceUnitStateData(state, key + ":" + j, true); + value = new ServiceUnitStateData(state, key + ":" + j, true, versionId); } else { - value = new ServiceUnitStateData(state, key + ":" + j, false); + value = new ServiceUnitStateData(state, key + ":" + j, false, versionId); } } @@ -560,15 +567,16 @@ public void testSlowTableviewAfterCompaction() throws Exception { String bundle = "bundle1"; String src = "broker0"; String dst = "broker1"; - producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src)).send(); + long versionId = 1; + producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src, versionId++)).send(); for (int i = 0; i < 3; i++) { - var assignedStateData = new ServiceUnitStateData(Assigning, dst, src); + var assignedStateData = new ServiceUnitStateData(Assigning, dst, src, versionId++); producer.newMessage().key(bundle).value(assignedStateData).send(); producer.newMessage().key(bundle).value(assignedStateData).send(); - var releasedStateData = new ServiceUnitStateData(Releasing, dst, src); + var releasedStateData = new ServiceUnitStateData(Releasing, dst, src, versionId++); producer.newMessage().key(bundle).value(releasedStateData).send(); producer.newMessage().key(bundle).value(releasedStateData).send(); - var ownedStateData = new ServiceUnitStateData(Owned, dst, src); + var ownedStateData = new ServiceUnitStateData(Owned, dst, src, versionId++); producer.newMessage().key(bundle).value(ownedStateData).send(); producer.newMessage().key(bundle).value(ownedStateData).send(); compactor.compact(topic, strategy).get(); @@ -945,4 +953,8 @@ public void testReadUnCompacted() assertNull(none); } } + + public static long versionId(ServiceUnitStateData data) { + return data == null ? ServiceUnitStateChannelImpl.VERSION_ID_INIT - 1 : data.versionId(); + } } From 579f22c8449be287ee1209a477aeaad346495289 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 1 Mar 2023 01:02:11 -0800 Subject: [PATCH 032/404] [improve][broker] PIP-192 Added --extensions option in BrokerMonitor (#19654) --- .../pulsar/testclient/BrokerMonitor.java | 94 +++++++++++++++++-- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java index c209c34a3d76e..3f8969860163d 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.testclient; +import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.BROKER_LOAD_DATA_STORE_TOPIC; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; @@ -31,7 +32,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SizeUnit; +import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; @@ -54,7 +61,7 @@ public class BrokerMonitor { private static final String BROKER_ROOT = "/loadbalance/brokers"; private static final int ZOOKEEPER_TIMEOUT_MILLIS = 30000; private static final int GLOBAL_STATS_PRINT_PERIOD_MILLIS = 60000; - private final ZooKeeper zkClient; + private ZooKeeper zkClient; private static final Gson gson = new Gson(); // Fields common for message rows. @@ -77,7 +84,7 @@ public class BrokerMonitor { private static final Object[] ALLOC_MESSAGE_ROW = makeMessageRow("ALLOC MSG"); private static final Object[] GLOBAL_HEADER = { "BROKER", "BUNDLE", "MSG/S", "LONG/S", "KB/S", "MAX %" }; - private final Map loadData; + private Map loadData; private static final FixedColumnLengthTableMaker localTableMaker = new FixedColumnLengthTableMaker(); static { @@ -434,8 +441,11 @@ private static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) boolean help; - @Parameter(names = { "--connect-string" }, description = "Zookeeper connect string", required = true) + @Parameter(names = { "--connect-string" }, description = "Zookeeper or broker connect string", required = true) public String connectString = null; + + @Parameter(names = { "--extensions" }, description = "true to monitor Load Balance Extensions.") + boolean extensions = false; } /** @@ -464,6 +474,71 @@ public void start() { } } + private TableView brokerLoadDataTableView; + + private BrokerMonitor(String brokerServiceUrl) { + try { + PulsarClient client = PulsarClient.builder() + .memoryLimit(0, SizeUnit.BYTES) + .serviceUrl(brokerServiceUrl) + .connectionsPerBroker(4) + .ioThreads(Runtime.getRuntime().availableProcessors()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + this.brokerLoadDataTableView = client + .newTableView(Schema.JSON(BrokerLoadData.class)) + .topic(BROKER_LOAD_DATA_STORE_TOPIC).create(); + } catch (Throwable e) { + log.info("Failed to start BrokerMonitor", e); + throw new RuntimeException(e); + } + } + + private synchronized void printBrokerLoadData(final String broker, final BrokerLoadData brokerLoadData) { + + // Initialize the constant rows. + final Object[][] rows = new Object[6][]; + rows[0] = SYSTEM_ROW; + rows[2] = COUNT_ROW; + rows[4] = LATEST_ROW; + + // First column is a label, so start at the second column at index 1. + // System row. + rows[1] = new Object[SYSTEM_ROW.length]; + initRow(rows[1], brokerLoadData.getCpu().percentUsage(), brokerLoadData.getMemory().percentUsage(), + brokerLoadData.getDirectMemory().percentUsage(), brokerLoadData.getBandwidthIn().percentUsage(), + brokerLoadData.getBandwidthOut().percentUsage(), brokerLoadData.getMaxResourceUsage() * 100); + + // Count row. + rows[3] = new Object[COUNT_ROW.length]; + initRow(rows[3], null, brokerLoadData.getBundleCount(), + null, null, + null, null); + + // Latest message data row. + rows[5] = new Object[LATEST_ROW.length]; + initMessageRow(rows[5], brokerLoadData.getMsgRateIn(), brokerLoadData.getMsgRateOut(), + brokerLoadData.getMsgThroughputIn(), brokerLoadData.getMsgThroughputOut()); + + final String table = localTableMaker.make(rows); + log.info("\nBroker Data for {}:\n{}\n", broker, table); + } + + private synchronized void printBrokerLoadDataStore() { + brokerLoadDataTableView.forEach(this::printBrokerLoadData); + } + + private void startBrokerLoadDataStoreMonitor() { + try { + while (true) { + Thread.sleep(GLOBAL_STATS_PRINT_PERIOD_MILLIS); + printBrokerLoadDataStore(); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + /** * Run a monitor from command line arguments. * @@ -481,8 +556,15 @@ public static void main(String[] args) throws Exception { jc.usage(); PerfClientUtils.exit(1); } - final ZooKeeper zkClient = new ZooKeeper(arguments.connectString, ZOOKEEPER_TIMEOUT_MILLIS, null); - final BrokerMonitor monitor = new BrokerMonitor(zkClient); - monitor.start(); + + + if (arguments.extensions) { + final BrokerMonitor monitor = new BrokerMonitor(arguments.connectString); + monitor.startBrokerLoadDataStoreMonitor(); + } else { + final ZooKeeper zkClient = new ZooKeeper(arguments.connectString, ZOOKEEPER_TIMEOUT_MILLIS, null); + final BrokerMonitor monitor = new BrokerMonitor(zkClient); + monitor.start(); + } } } From 7e0a6c48d23a3706182977c2e47eea211306ab5a Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Wed, 1 Mar 2023 22:33:59 +0800 Subject: [PATCH 033/404] [fix][broker] Fixed history load not releasing (#19560) When a broker in the cluster is not active, its load still remain in the memory of the Leader. If deployed in a container, brokers will scale up and down frequently, and the memory of the Leader gradually increase --- .../loadbalance/LoadSheddingStrategy.java | 8 ++++++++ .../impl/ModularLoadManagerImpl.java | 3 +++ .../loadbalance/impl/ThresholdShedder.java | 9 +++++++-- .../loadbalance/impl/ThresholdShedderTest.java | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java index ee136d6e9b851..3cad4b74d9903 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance; import com.google.common.collect.Multimap; +import java.util.Set; import org.apache.pulsar.broker.ServiceConfiguration; /** @@ -36,4 +37,11 @@ public interface LoadSheddingStrategy { * @return A map from all selected bundles to the brokers on which they reside. */ Multimap findBundlesForUnloading(LoadData loadData, ServiceConfiguration conf); + + /** + * Triggered when active broker changes. + * + * @param activeBrokers active Brokers + */ + default void onActiveBrokersChange(Set activeBrokers) {} } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index c1afe6007f6bb..ea2472eb199d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -491,6 +491,9 @@ private void cleanupDeadBrokersData() { if (pulsar.getLeaderElectionService() != null && pulsar.getLeaderElectionService().isLeader()) { deadBrokers.forEach(this::deleteTimeAverageDataFromMetadataStoreAsync); + for (LoadSheddingStrategy loadSheddingStrategy : loadSheddingPipeline) { + loadSheddingStrategy.onActiveBrokersChange(activeBrokers); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index e2f1a7808fe91..86df49f952674 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -22,6 +22,7 @@ import com.google.common.collect.Multimap; import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableDouble; import org.apache.commons.lang3.tuple.Pair; @@ -59,7 +60,8 @@ public class ThresholdShedder implements LoadSheddingStrategy { private final Map brokerAvgResourceUsage = new HashMap<>(); @Override - public Multimap findBundlesForUnloading(final LoadData loadData, final ServiceConfiguration conf) { + public synchronized Multimap findBundlesForUnloading(final LoadData loadData, + final ServiceConfiguration conf) { selectedBundlesCache.clear(); final double threshold = conf.getLoadBalancerBrokerThresholdShedderPercentage() / 100.0; final Map recentlyUnloadedBundles = loadData.getRecentlyUnloadedBundles(); @@ -75,7 +77,6 @@ public Multimap findBundlesForUnloading(final LoadData loadData, loadData.getBrokerData().forEach((broker, brokerData) -> { final LocalBrokerData localData = brokerData.getLocalData(); final double currentUsage = brokerAvgResourceUsage.getOrDefault(broker, 0.0); - if (currentUsage < avgUsage + threshold) { if (log.isDebugEnabled()) { log.debug("[{}] broker is not overloaded, ignoring at this point", broker); @@ -233,5 +234,9 @@ private Pair getMaxUsageBroker( } return Pair.of(hasBrokerBelowLowerBound, maxUsageBrokerName); } + @Override + public synchronized void onActiveBrokersChange(Set newBrokers) { + brokerAvgResourceUsage.keySet().removeIf((key) -> !newBrokers.contains(key)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java index 4747471eab09f..16a15f5a7c7f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java @@ -23,7 +23,10 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import java.lang.reflect.Field; +import java.util.HashSet; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadData; @@ -57,6 +60,21 @@ public void testNoBrokers() { assertTrue(thresholdShedder.findBundlesForUnloading(loadData, conf).isEmpty()); } + @Test + public void testCleanCache() throws Exception { + testBrokerReachThreshold(); + Field field = ThresholdShedder.class.getDeclaredField("brokerAvgResourceUsage"); + field.setAccessible(true); + Map map = (Map) field.get(thresholdShedder); + assertFalse(map.isEmpty()); + HashSet activeBrokers = new HashSet<>(); + activeBrokers.add("leader"); + thresholdShedder.onActiveBrokersChange(activeBrokers); + thresholdShedder.findBundlesForUnloading(new LoadData(), conf); + map = (Map) field.get(thresholdShedder); + assertTrue(map.isEmpty()); + } + @Test public void testBrokersWithNoBundles() { LoadData loadData = new LoadData(); From 6621fd3451a52bb7447ebeb845c83ac2f310b718 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 1 Mar 2023 11:45:26 -0600 Subject: [PATCH 034/404] [improve] Simplify enabling Broker, WS Proxy hostname verification (#19674) ### Motivation When we merged #15818 in order to make the broker's client configurable, we did not add an explicit config for hostname verification. This PR adds that config to the broker and the websocket proxy. I chose the name `tlsHostnameVerificationEnabled` because that is what is already used in the proxy. It diverges from the function worker's config of `tlsEnableHostnameVerification`. Before this PR, you would have enabled hostname verification by configuring `brokerClient_tlsHostnameVerificationEnable=true` in the broker and WS proxy configs. (Note that the variable name is slightly different because the `ClientConfiguration` does not have a `d` at the end of its name. The remaining follow up work will be to update the `ClusterData` objects to configure hostname verification there to make it easier to configure hostname verification for remote clusters. ### Modifications * Add `tlsHostnameVerificationEnabled` to the `broker.conf` and the `proxy.conf` * Update all of the relevant locations that were previously only relying on `brokerClient_tlsHostnameVerificationEnable` ### Verifying this change I added a single test to ensure that the `WebSocketProxyConfiguration` properly converts to the `ServiceConfiguration` object. Otherwise, I verified that anywhere we are using `"brokerClient_"`, this PR also adds the right configuration. ### Does this pull request potentially affect one of the following parts: This PR introduces a "new" configuration key, but not a new concept. All underlying behaviors are unchanged. ### Documentation - [x] `doc-not-needed` Docs are automatically updated by these changes. --- conf/broker.conf | 6 +++++ conf/websocket.conf | 3 +++ .../pulsar/broker/ServiceConfiguration.java | 5 ++++ .../apache/pulsar/broker/PulsarService.java | 6 +++-- .../broker/namespace/NamespaceService.java | 3 ++- .../pulsar/broker/service/BrokerService.java | 24 ++++++++++++------- .../pulsar/compaction/CompactorTool.java | 3 ++- .../pulsar/websocket/WebSocketService.java | 1 + .../service/WebSocketProxyConfiguration.java | 6 ++++- .../WebSocketProxyConfigurationTest.java | 16 +++++++++++++ 10 files changed, 60 insertions(+), 13 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 3183c83a837a8..4b7c108be5f11 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -680,6 +680,9 @@ tlsTrustCertsFilePath= # though the cert will not be used for client authentication. tlsAllowInsecureConnection=false +# Whether the hostname is validated when the broker creates a TLS connection with other brokers +tlsHostnameVerificationEnabled=false + # Specify the tls protocols the broker will use to negotiate during TLS handshake # (a comma-separated list of protocol names). # Examples: @@ -1055,6 +1058,9 @@ bookkeeperTLSTrustCertsFilePath= # Tls cert refresh duration at bookKeeper-client in seconds (0 to disable check) bookkeeperTlsCertFilesRefreshDurationSeconds=300 +# Whether the hostname is validated when the broker creates a TLS connection to a bookkeeper +bookkeeper_tlsHostnameVerificationEnabled=false + # Enable/disable disk weight based placement. Default is false bookkeeperDiskWeightBasedPlacementEnabled=false diff --git a/conf/websocket.conf b/conf/websocket.conf index a966f05c71935..2e2824a838c6f 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -108,6 +108,9 @@ brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= brokerClientTrustCertsFilePath= +# Whether the hostname is validated when connecting to the broker. +tlsHostnameVerificationEnabled=false + # You can add extra configuration options for the Pulsar Client # by prefixing them with "brokerClient_". These configurations are applied after hard coded configuration # and before the above brokerClient configurations named above. diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index c18367f265506..ec5b0d4042bcd 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1490,6 +1490,11 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se doc = "Accept untrusted TLS certificate from client" ) private boolean tlsAllowInsecureConnection = false; + @FieldContext( + category = CATEGORY_TLS, + doc = "Whether the hostname is validated when the broker creates a TLS connection with other brokers" + ) + private boolean tlsHostnameVerificationEnabled = false; @FieldContext( category = CATEGORY_TLS, doc = "Specify the tls protocols the broker will use to negotiate during TLS Handshake.\n\n" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index ae0fb1a9f283c..4af94c339c82f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1551,6 +1551,7 @@ public synchronized PulsarClient getClient() throws PulsarServerException { conf.setTlsCiphers(this.getConfiguration().getBrokerClientTlsCiphers()); conf.setTlsProtocols(this.getConfiguration().getBrokerClientTlsProtocols()); conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); + conf.setTlsHostnameVerificationEnable(this.getConfiguration().isTlsHostnameVerificationEnabled()); if (this.getConfiguration().isBrokerClientTlsEnabledWithKeyStore()) { conf.setUseKeyStoreTls(true); conf.setTlsTrustStoreType(this.getConfiguration().getBrokerClientTlsTrustStoreType()); @@ -1622,7 +1623,8 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { .tlsKeyFilePath(conf.getBrokerClientKeyFilePath()) .tlsCertificateFilePath(conf.getBrokerClientCertificateFilePath()); } - builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()); + builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()) + .enableTlsHostnameVerification(conf.isTlsHostnameVerificationEnabled()); } // most of the admin request requires to make zk-call so, keep the max read-timeout based on @@ -1849,7 +1851,7 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu workerConfig.setMetadataStoreCacheExpirySeconds(brokerConfig.getMetadataStoreCacheExpirySeconds()); workerConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); workerConfig.setTlsEnabled(brokerConfig.isTlsEnabled()); - workerConfig.setTlsEnableHostnameVerification(false); + workerConfig.setTlsEnableHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath()); // client in worker will use this config to authenticate with broker diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 245c3f896af1f..33b15926c3c33 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1420,7 +1420,8 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { ? cluster.getBrokerServiceUrlTls() : cluster.getServiceUrlTls()) .enableTls(true) .tlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath()) - .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection()); + .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection()) + .enableTlsHostnameVerification(pulsar.getConfiguration().isTlsHostnameVerificationEnabled()); } else { clientBuilder.serviceUrl(isNotBlank(cluster.getBrokerServiceUrl()) ? cluster.getBrokerServiceUrl() : cluster.getServiceUrl()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index b9b63427b370e..15d48e59849b6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1307,7 +1307,8 @@ public PulsarClient getReplicationClient(String cluster, Optional c data.getBrokerClientTlsKeyStorePassword(), data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), - data.getBrokerClientCertificateFilePath() + data.getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } else if (pulsar.getConfiguration().isBrokerClientTlsEnabled()) { configTlsSettings(clientBuilder, serviceUrlTls, @@ -1321,7 +1322,8 @@ public PulsarClient getReplicationClient(String cluster, Optional c pulsar.getConfiguration().getBrokerClientTlsKeyStorePassword(), pulsar.getConfiguration().getBrokerClientTrustCertsFilePath(), pulsar.getConfiguration().getBrokerClientKeyFilePath(), - pulsar.getConfiguration().getBrokerClientCertificateFilePath() + pulsar.getConfiguration().getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } else { clientBuilder.serviceUrl( @@ -1351,10 +1353,12 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, String brokerClientTlsTrustStorePassword, String brokerClientTlsKeyStoreType, String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, - String brokerClientKeyFilePath, String brokerClientCertificateFilePath) { + String brokerClientKeyFilePath, String brokerClientCertificateFilePath, + boolean isTlsHostnameVerificationEnabled) { clientBuilder .serviceUrl(serviceUrl) - .allowTlsInsecureConnection(isTlsAllowInsecureConnection); + .allowTlsInsecureConnection(isTlsAllowInsecureConnection) + .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); if (brokerClientTlsEnabledWithKeyStore) { clientBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1376,7 +1380,8 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro String brokerClientTlsTrustStorePassword, String brokerClientTlsKeyStoreType, String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, - String brokerClientKeyFilePath, String brokerClientCertificateFilePath) { + String brokerClientKeyFilePath, String brokerClientCertificateFilePath, + boolean isTlsHostnameVerificationEnabled) { if (brokerClientTlsEnabledWithKeyStore) { adminBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1390,7 +1395,8 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro .tlsKeyFilePath(brokerClientKeyFilePath) .tlsCertificateFilePath(brokerClientCertificateFilePath); } - adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection); + adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection) + .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); } public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { @@ -1438,7 +1444,8 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c data.getBrokerClientTlsKeyStorePassword(), data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), - data.getBrokerClientCertificateFilePath() + data.getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } else if (conf.isBrokerClientTlsEnabled()) { configAdminTlsSettings(builder, @@ -1452,7 +1459,8 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c conf.getBrokerClientTlsKeyStorePassword(), conf.getBrokerClientTrustCertsFilePath(), conf.getBrokerClientKeyFilePath(), - conf.getBrokerClientCertificateFilePath() + conf.getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index c359823549fcd..2539c306500a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -82,7 +82,8 @@ public static PulsarClient createClient(ServiceConfiguration brokerConfig) throw AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar+ssl"); if (internalListener.getBrokerServiceUrlTls() != null && brokerConfig.isBrokerClientTlsEnabled()) { clientBuilder.serviceUrl(internalListener.getBrokerServiceUrlTls().toString()) - .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); + .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()) + .enableTlsHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); if (brokerConfig.isBrokerClientTlsEnabledWithKeyStore()) { clientBuilder.useKeyStoreTls(true) .tlsKeyStoreType(brokerConfig.getBrokerClientTlsKeyStoreType()) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java index 9a8653029ce4c..66b2a0075ec2d 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java @@ -199,6 +199,7 @@ private PulsarClient createClientInstance(ClusterData clusterData) throws IOExce .statsInterval(0, TimeUnit.SECONDS) // .enableTls(config.isTlsEnabled()) // .allowTlsInsecureConnection(config.isTlsAllowInsecureConnection()) // + .enableTlsHostnameVerification(config.isTlsHostnameVerificationEnabled()) .tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath()) // .ioThreads(config.getWebSocketNumIoThreads()) // .connectionsPerBroker(config.getWebSocketConnectionsPerBroker()); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java index 7dba9e719ad5a..7acfd4a64ad35 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java @@ -129,6 +129,10 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "Path for the trusted TLS certificate file for outgoing connection to a server (broker)") private String brokerClientTrustCertsFilePath = ""; + // Note: name matches the ServiceConfiguration name to ensure correct mapping + @FieldContext(doc = "Enable TLS hostname verification when connecting to broker") + private boolean tlsHostnameVerificationEnabled = false; + @FieldContext(doc = "Number of IO threads in Pulsar client used in WebSocket proxy") private int webSocketNumIoThreads = Runtime.getRuntime().availableProcessors(); @@ -183,7 +187,7 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "Path for the trusted TLS certificate file") private String tlsTrustCertsFilePath = ""; - @FieldContext(doc = "Accept untrusted TLS certificate from client") + @FieldContext(doc = "Accept untrusted TLS certificate from client and broker") private boolean tlsAllowInsecureConnection = false; @FieldContext(doc = "Specify whether client certificates are required for " diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java index e3a303003b841..92b4238cddd7d 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java @@ -19,9 +19,11 @@ package org.apache.pulsar.websocket; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; import org.testng.annotations.Test; +import org.testng.Assert; import java.io.File; import java.io.FileInputStream; @@ -87,4 +89,18 @@ public void testBackwardCompatibility() throws IOException { assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 100); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 300); } + + @Test + public void testConfigurationConversionUsedByWebSocketProxyStarter() { + WebSocketProxyConfiguration config = new WebSocketProxyConfiguration(); + // Use non-default values for testing + config.setTlsAllowInsecureConnection(true); + Assert.assertFalse(config.isTlsHostnameVerificationEnabled(), "Update me when default changes."); + config.setTlsHostnameVerificationEnabled(true); + ServiceConfiguration brokerConf = PulsarConfigurationLoader.convertFrom(config); + Assert.assertTrue(brokerConf.isTlsAllowInsecureConnection(), + "TlsAllowInsecureConnection should convert correctly"); + Assert.assertTrue(brokerConf.isTlsHostnameVerificationEnabled(), + "TlsHostnameVerificationEnabled should convert correctly"); + } } From 5122f7721b236e763c72193fabc064ace8ab7399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 1 Mar 2023 19:31:56 +0100 Subject: [PATCH 035/404] [fix][broker] Fix LedgerOffloaderStatsImpl singleton close method (#19666) --- .../impl/LedgerOffloaderStatsImpl.java | 6 +++-- .../stats/LedgerOffloaderMetricsTest.java | 23 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java index be2895bf81867..5e05e4c8137cd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java @@ -276,17 +276,19 @@ public void run() { } @Override - public void close() throws Exception { + public synchronized void close() throws Exception { if (instance == this && this.closed.compareAndSet(false, true)) { CollectorRegistry.defaultRegistry.unregister(this.offloadError); CollectorRegistry.defaultRegistry.unregister(this.offloadRate); CollectorRegistry.defaultRegistry.unregister(this.readLedgerLatency); CollectorRegistry.defaultRegistry.unregister(this.writeStorageError); CollectorRegistry.defaultRegistry.unregister(this.readOffloadError); + CollectorRegistry.defaultRegistry.unregister(this.readOffloadBytes); CollectorRegistry.defaultRegistry.unregister(this.readOffloadRate); CollectorRegistry.defaultRegistry.unregister(this.readOffloadIndexLatency); CollectorRegistry.defaultRegistry.unregister(this.readOffloadDataLatency); - this.offloadAndReadOffloadBytesMap.clear(); + CollectorRegistry.defaultRegistry.unregister(this.deleteOffloadOps); + instance = null; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java index a285974f13dcc..effb12a548e3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.stats; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -32,7 +31,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -public class LedgerOffloaderMetricsTest extends BrokerTestBase { +public class LedgerOffloaderMetricsTest extends BrokerTestBase { @Override protected void setup() throws Exception { @@ -60,7 +59,7 @@ public void testTopicLevelMetrics() throws Exception { String ns1 = "prop/ns-abc1"; admin.namespaces().createNamespace(ns1); - String []topics = new String[3]; + String[] topics = new String[3]; LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) pulsar.getOffloaderStats(); for (int i = 0; i < 3; i++) { @@ -80,10 +79,10 @@ public void testTopicLevelMetrics() throws Exception { for (String topicName : topics) { assertEquals(offloaderStats.getOffloadError(topicName), 2); - assertEquals(offloaderStats.getOffloadBytes(topicName) , 100); + assertEquals(offloaderStats.getOffloadBytes(topicName), 100); assertEquals((long) offloaderStats.getReadLedgerLatency(topicName).sum, 1); assertEquals(offloaderStats.getReadOffloadError(topicName), 2); - assertEquals((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum ,1000); + assertEquals((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum, 1000); assertEquals(offloaderStats.getReadOffloadBytes(topicName), 100000); assertEquals(offloaderStats.getWriteStorageError(topicName), 2); } @@ -126,13 +125,13 @@ public void testNamespaceLevelMetrics() throws Exception { List topics = entry.getValue(); String topicName = topics.get(0); - assertTrue(offloaderStats.getOffloadError(topicName) >= 1); - assertTrue(offloaderStats.getOffloadBytes(topicName) >= 100); - assertTrue((long) offloaderStats.getReadLedgerLatency(topicName).sum >= 1); - assertTrue(offloaderStats.getReadOffloadError(topicName) >= 1); - assertTrue((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum >= 1000); - assertTrue(offloaderStats.getReadOffloadBytes(topicName) >= 100000); - assertTrue(offloaderStats.getWriteStorageError(topicName) >= 1); + assertEquals(offloaderStats.getOffloadError(topicName), 6); + assertEquals(offloaderStats.getOffloadBytes(topicName), 600); + assertEquals((long) offloaderStats.getReadLedgerLatency(topicName).sum, 6); + assertEquals(offloaderStats.getReadOffloadError(topicName), 6); + assertEquals((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum, 6000); + assertEquals(offloaderStats.getReadOffloadBytes(topicName), 600000); + assertEquals(offloaderStats.getWriteStorageError(topicName), 6); } } From 7f37670fc96e4b629234c242257fe1d42480e5cd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 1 Mar 2023 13:36:12 -0600 Subject: [PATCH 036/404] [fix][client] Fix typo in Java doc for ClientConfigurationData (#19672) ### Motivation A minor typo fix. ### Modifications Update `ClientConfigurationData` by replacing `proxy` with `client`. ### Documentation - [x] `doc` This is a doc change. --- .../apache/pulsar/client/impl/conf/ClientConfigurationData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 481bdc99f9a99..1e7bc6f8221cf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -170,7 +170,7 @@ public class ClientConfigurationData implements Serializable, Cloneable { @ApiModelProperty( name = "tlsHostnameVerificationEnable", - value = "Whether the hostname is validated when the proxy creates a TLS connection with brokers." + value = "Whether the hostname is validated when the client creates a TLS connection with brokers." ) private boolean tlsHostnameVerificationEnable = false; @ApiModelProperty( From 5e717c68e061562fd44f2c44c4a41667abaf4df8 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 2 Mar 2023 09:48:22 +0800 Subject: [PATCH 037/404] [improve][broker] Make BucketDelayedDeliveryTracker can retry snapshot operation & improve logs (#19577) --- .../bookkeeper/mledger/util/Futures.java | 4 +- .../BookkeeperBucketSnapshotStorage.java | 12 +- .../pulsar/broker/delayed/bucket/Bucket.java | 25 +- .../bucket/BucketDelayedDeliveryTracker.java | 220 ++++++++++++------ .../delayed/bucket/ImmutableBucket.java | 152 +++++++----- .../broker/delayed/bucket/MutableBucket.java | 20 +- .../delayed/MockBucketSnapshotStorage.java | 41 ++++ .../broker/delayed/MockManagedCursor.java | 2 +- .../BucketDelayedDeliveryTrackerTest.java | 117 +++++++++- 9 files changed, 433 insertions(+), 160 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java index dc1d1eb6c9ac5..f5ad77a71d8c4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java @@ -24,6 +24,7 @@ import java.util.function.Supplier; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.pulsar.common.util.FutureUtil; /** * Conveniences to use with {@link CompletableFuture}. @@ -78,7 +79,8 @@ public static CompletableFuture executeWithRetry(Supplier 0) { + Throwable throwable = FutureUtil.unwrapCompletionException(ex); + if (needRetryExceptionClass.isAssignableFrom(throwable.getClass()) && maxRetryTimes > 0) { executeWithRetry(op, needRetryExceptionClass, maxRetryTimes - 1).whenComplete((res2, ex2) -> { if (ex2 == null) { resultFuture.complete(res2); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 1cadc6d98e268..7dd6266e2115c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -154,7 +154,7 @@ private CompletableFuture createLedger(String bucketKey) { LedgerPassword, (rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to create ledger", rc, -1)); + future.completeExceptionally(bkException("Create ledger", rc, -1)); } else { future.complete(handle); } @@ -170,7 +170,7 @@ private CompletableFuture openLedger(Long ledgerId) { LedgerPassword, (rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to open ledger", rc, ledgerId)); + future.completeExceptionally(bkException("Open ledger", rc, ledgerId)); } else { future.complete(handle); } @@ -184,7 +184,7 @@ private CompletableFuture closeLedger(LedgerHandle ledgerHandle) { ledgerHandle.asyncClose((rc, handle, ctx) -> { if (rc != BKException.Code.OK) { log.warn("Failed to close a Ledger Handle: {}", ledgerHandle.getId()); - future.completeExceptionally(bkException("Failed to close ledger", rc, ledgerHandle.getId())); + future.completeExceptionally(bkException("Close ledger", rc, ledgerHandle.getId())); } else { future.complete(null); } @@ -197,7 +197,7 @@ private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data) ledgerHandle.asyncAddEntry(data, (rc, handle, entryId, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to add entry", rc, ledgerHandle.getId())); + future.completeExceptionally(bkException("Add entry", rc, ledgerHandle.getId())); } else { future.complete(null); } @@ -217,7 +217,7 @@ CompletableFuture> getLedgerEntryThenCloseLedger(Ledger ledger.asyncReadEntries(firstEntryId, lastEntryId, (rc, handle, entries, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to read entry", rc, ledger.getId())); + future.completeExceptionally(bkException("Read entry", rc, ledger.getId())); } else { future.complete(entries); } @@ -231,7 +231,7 @@ private CompletableFuture deleteLedger(long ledgerId) { CompletableFuture future = new CompletableFuture<>(); bookKeeper.asyncDeleteLedger(ledgerId, (int rc, Object cnx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to delete ledger", rc, ledgerId)); + future.completeExceptionally(bkException("Delete ledger", rc, ledgerId)); } else { future.complete(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 5d2a556337a6e..50b5cd12ead07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -42,6 +42,8 @@ abstract class Bucket { static final String DELIMITER = "_"; static final int MaxRetryTimes = 3; + protected final String dispatcherName; + protected final ManagedCursor cursor; protected final BucketSnapshotStorage bucketSnapshotStorage; @@ -54,17 +56,18 @@ abstract class Bucket { int lastSegmentEntryId; - int currentSegmentEntryId; + volatile int currentSegmentEntryId; - long snapshotLength; + volatile long snapshotLength; private volatile Long bucketId; private volatile CompletableFuture snapshotCreateFuture; - Bucket(ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - this(cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); + Bucket(String dispatcherName, ManagedCursor cursor, + BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { + this(dispatcherName, cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); } boolean containsMessage(long ledgerId, long entryId) { @@ -126,13 +129,19 @@ CompletableFuture asyncSaveBucketSnapshot( ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments) { final String bucketKey = bucket.bucketKey(); - return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey) - .thenCompose(newBucketId -> { + return executeWithRetry( + () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey) + .whenComplete((__, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to create bucket snapshot, bucketKey: {}", + dispatcherName, bucketKey, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes).thenCompose(newBucketId -> { bucket.setBucketId(newBucketId); return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { - log.warn("Failed to record bucketId to cursor property, bucketKey: {}, bucketId: {}", - bucketKey, bucketId); + log.warn("[{}] Failed to record bucketId to cursor property, bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey, newBucketId, ex); return null; }).thenApply(__ -> newBucketId); }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index bd4ef92cc7320..a77b272297be4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELAYED_BUCKET_KEY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; +import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Range; @@ -61,7 +62,9 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker { - static final int AsyncOperationTimeoutSeconds = 30; + static final CompletableFuture NULL_LONG_PROMISE = CompletableFuture.completedFuture(null); + + static final int AsyncOperationTimeoutSeconds = 60; private final long minIndexCountPerBucket; @@ -104,20 +107,19 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); this.snapshotSegmentLastIndexTable = HashBasedTable.create(); - ManagedCursor cursor = dispatcher.getCursor(); - this.lastMutableBucket = new MutableBucket(cursor, bucketSnapshotStorage); + this.lastMutableBucket = new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), bucketSnapshotStorage); this.numberDelayedMessages = recoverBucketSnapshot(); } private synchronized long recoverBucketSnapshot() throws RuntimeException { - ManagedCursor cursor = this.lastMutableBucket.cursor; + ManagedCursor cursor = this.lastMutableBucket.getCursor(); Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>(); cursor.getCursorProperties().keySet().forEach(key -> { if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { String[] keys = key.split(DELIMITER); checkArgument(keys.length == 3); ImmutableBucket immutableBucket = - new ImmutableBucket(cursor, this.lastMutableBucket.bucketSnapshotStorage, + new ImmutableBucket(dispatcher.getName(), cursor, this.lastMutableBucket.bucketSnapshotStorage, Long.parseLong(keys[1]), Long.parseLong(keys[2])); putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), immutableBucket, toBeDeletedBucketMap); @@ -126,6 +128,8 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges(); if (immutableBucketMap.isEmpty()) { + log.info("[{}] Recover delayed message index bucket snapshot finish, don't find bucket snapshot", + dispatcher.getName()); return 0; } @@ -232,10 +236,42 @@ private void afterCreateImmutableBucket(Pair immu DelayedIndex lastDelayedIndex = immutableBucketDelayedIndexPair.getRight(); snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), immutableBucket); - if (log.isDebugEnabled()) { - log.debug("[{}] Create bucket snapshot, bucket: {}", dispatcher.getName(), - lastMutableBucket); - } + + immutableBucket.getSnapshotCreateFuture().ifPresent(createFuture -> { + CompletableFuture future = createFuture.whenComplete((__, ex) -> { + if (ex == null) { + immutableBucket.setSnapshotSegments(null); + log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(), + immutableBucket.bucketKey()); + return; + } + + //TODO Record create snapshot failed + log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", + dispatcher.getName(), immutableBucket.bucketKey(), ex); + + // Put indexes back into the shared queue and downgrade to memory mode + synchronized (BucketDelayedDeliveryTracker.this) { + immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> { + for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment : + snapshotSegments) { + for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) { + sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(), + delayedIndex.getLedgerId(), delayedIndex.getEntryId()); + } + } + immutableBucket.setSnapshotSegments(null); + }); + + immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId); + immutableBuckets.remove( + Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId)); + snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getTimestamp()); + } + }); + immutableBucket.setSnapshotCreateFuture(future); + }); } } @@ -263,12 +299,10 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { try { - asyncMergeBucketSnapshot().get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException(e); + asyncMergeBucketSnapshot().get(2 * AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); + } catch (Exception e) { + // Ignore exception to merge bucket on the next schedule. + log.error("[{}] An exception occurs when merge bucket snapshot.", dispatcher.getName(), e); } } } @@ -304,7 +338,10 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; if (numberMessages < minNumberMessages) { minNumberMessages = (int) numberMessages; - if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()) { + if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId() + && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId() + && bucketL.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone() + && bucketR.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()) { minIndex = i; } } @@ -314,62 +351,66 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { log.warn("[{}] Can't find able merged bucket", dispatcher.getName()); return CompletableFuture.completedFuture(null); } - return asyncMergeBucketSnapshot(values.get(minIndex), values.get(minIndex + 1)); + + ImmutableBucket immutableBucketA = values.get(minIndex); + ImmutableBucket immutableBucketB = values.get(minIndex + 1); + + if (log.isDebugEnabled()) { + log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(), + immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); + } + return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> { + if (ex != null) { + log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}", + dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex); + } else { + log.info("[{}] Merge bucket snapshot finish, bucketAKey: {}, bucketBKey: {}", + dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); + } + }); } private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA, ImmutableBucket bucketB) { - CompletableFuture snapshotCreateFutureA = - bucketA.getSnapshotCreateFuture().orElse(CompletableFuture.completedFuture(null)); - CompletableFuture snapshotCreateFutureB = - bucketB.getSnapshotCreateFuture().orElse(CompletableFuture.completedFuture(null)); - - return CompletableFuture.allOf(snapshotCreateFutureA, snapshotCreateFutureB).thenCompose(__ -> { - CompletableFuture> futureA = - bucketA.getRemainSnapshotSegment(); - CompletableFuture> futureB = - bucketB.getRemainSnapshotSegment(); - return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) - .thenAccept(combinedDelayedIndexQueue -> { - Pair immutableBucketDelayedIndexPair = - lastMutableBucket.createImmutableBucketAndAsyncPersistent( - timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue, - combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId); - - // Merge bit map to new bucket - Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); - Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); - Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); - delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { - delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { - if (bitMapA == null) { - return bitMapB; - } - - bitMapA.or(bitMapB); - return bitMapA; - }); + CompletableFuture> futureA = + bucketA.getRemainSnapshotSegment(); + CompletableFuture> futureB = + bucketB.getRemainSnapshotSegment(); + return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) + .thenAccept(combinedDelayedIndexQueue -> { + Pair immutableBucketDelayedIndexPair = + lastMutableBucket.createImmutableBucketAndAsyncPersistent( + timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue, + combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId); + + // Merge bit map to new bucket + Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); + Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); + Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); + delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { + delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { + if (bitMapA == null) { + return bitMapB; + } + + bitMapA.or(bitMapB); + return bitMapA; }); - immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); - - afterCreateImmutableBucket(immutableBucketDelayedIndexPair); - - CompletableFuture snapshotCreateFuture = CompletableFuture.completedFuture(null); - if (immutableBucketDelayedIndexPair != null) { - snapshotCreateFuture = immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() - .orElse(CompletableFuture.completedFuture(null)); - } + }); + immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); - snapshotCreateFuture.thenCompose(___ -> { - CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); - CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); - return CompletableFuture.allOf(removeAFuture, removeBFuture); - }); + afterCreateImmutableBucket(immutableBucketDelayedIndexPair); - immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() + .orElse(NULL_LONG_PROMISE).thenCompose(___ -> { + CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); + CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); + return CompletableFuture.allOf(removeAFuture, removeBFuture); }); - }); + + immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); + immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + }); } @Override @@ -422,19 +463,31 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa long ledgerId = sharedBucketPriorityQueue.peekN2(); long entryId = sharedBucketPriorityQueue.peekN3(); - positions.add(new PositionImpl(ledgerId, entryId)); - sharedBucketPriorityQueue.pop(); - removeIndexBit(ledgerId, entryId); - - ImmutableBucket bucket = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId); if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) { + final int preSegmentEntryId = bucket.currentSegmentEntryId; if (log.isDebugEnabled()) { - log.debug("[{}] Load next snapshot segment, bucket: {}", dispatcher.getName(), bucket); + log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); } // All message of current snapshot segment are scheduled, load next snapshot segment // TODO make it asynchronous and not blocking this process try { + boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); + + if (!createFutureDone) { + log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey()); + break; + } + + if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) { + immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + bucket.asyncDeleteBucketSnapshot(); + continue; + } + bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> { if (CollectionUtils.isEmpty(indexList)) { immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); @@ -449,13 +502,32 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } - }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - // TODO make this segment load again - throw new RuntimeException(e); + }).whenComplete((__, ex) -> { + if (ex != null) { + // Back bucket state + bucket.setCurrentSegmentEntryId(preSegmentEntryId); + + log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}", + dispatcher.getName(), bucket.bucketKey(), ex); + } else { + log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId); + } + }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); + snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + } catch (Exception e) { + // Ignore exception to reload this segment on the next schedule. + log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey(), e); + break; } } + positions.add(new PositionImpl(ledgerId, entryId)); + + sharedBucketPriorityQueue.pop(); + removeIndexBit(ledgerId, entryId); + --n; --numberDelayedMessages; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 3e9c577454fed..c947e53843a85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -18,14 +18,17 @@ */ package org.apache.pulsar.broker.delayed.bucket; +import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds; import com.google.protobuf.ByteString; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.CollectionUtils; @@ -38,8 +41,17 @@ @Slf4j class ImmutableBucket extends Bucket { - ImmutableBucket(ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - super(cursor, storage, startLedgerId, endLedgerId); + + @Setter + private volatile List snapshotSegments; + + ImmutableBucket(String dispatcherName, ManagedCursor cursor, + BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { + super(dispatcherName, cursor, storage, startLedgerId, endLedgerId); + } + + public Optional> getSnapshotSegments() { + return Optional.ofNullable(snapshotSegments); } CompletableFuture> asyncLoadNextBucketSnapshotEntry() { @@ -52,61 +64,65 @@ CompletableFuture> asyncRecoverBucketSnapshotEntry(Supplier> asyncLoadNextBucketSnapshotEntry(boolean isRecover, Supplier cutoffTimeSupplier) { - if (log.isDebugEnabled()) { - log.debug("[{}] Load next bucket snapshot data, bucket: {}", cursor.getName(), this); + final long bucketId = getAndUpdateBucketId(); + final CompletableFuture loadMetaDataFuture; + if (isRecover) { + final long cutoffTime = cutoffTimeSupplier.get(); + // Load Metadata of bucket snapshot + final String bucketKey = bucketKey(); + loadMetaDataFuture = executeWithRetry(() -> bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId) + .whenComplete((___, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to get bucket snapshot metadata," + + " bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey, bucketId, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes) + .thenApply(snapshotMetadata -> { + List metadataList = + snapshotMetadata.getMetadataListList(); + + // Skip all already reach schedule time snapshot segments + int nextSnapshotEntryIndex = 0; + while (nextSnapshotEntryIndex < metadataList.size() + && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { + nextSnapshotEntryIndex++; + } + + this.setLastSegmentEntryId(metadataList.size()); + this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); + + return nextSnapshotEntryIndex + 1; + }); + } else { + loadMetaDataFuture = CompletableFuture.completedFuture(currentSegmentEntryId + 1); } - // Wait bucket snapshot create finish - CompletableFuture snapshotCreateFuture = - getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(null)) - .thenApply(__ -> null); - - return snapshotCreateFuture.thenCompose(__ -> { - final long bucketId = getAndUpdateBucketId(); - final CompletableFuture loadMetaDataFuture; - if (isRecover) { - final long cutoffTime = cutoffTimeSupplier.get(); - // Load Metadata of bucket snapshot - loadMetaDataFuture = bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId) - .thenApply(snapshotMetadata -> { - List metadataList = - snapshotMetadata.getMetadataListList(); - - // Skip all already reach schedule time snapshot segments - int nextSnapshotEntryIndex = 0; - while (nextSnapshotEntryIndex < metadataList.size() - && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { - nextSnapshotEntryIndex++; - } - - this.setLastSegmentEntryId(metadataList.size()); - this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); - - return nextSnapshotEntryIndex + 1; - }); - } else { - loadMetaDataFuture = CompletableFuture.completedFuture(currentSegmentEntryId + 1); + return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { + if (nextSegmentEntryId > lastSegmentEntryId) { + return CompletableFuture.completedFuture(null); } - return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { - if (nextSegmentEntryId > lastSegmentEntryId) { - return CompletableFuture.completedFuture(null); - } - - return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) - .thenApply(bucketSnapshotSegments -> { - if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { - return Collections.emptyList(); - } - - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - bucketSnapshotSegments.get(0); - List indexList = - snapshotSegment.getIndexesList(); - this.setCurrentSegmentEntryId(nextSegmentEntryId); - return indexList; - }); - }); + return executeWithRetry( + () -> bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, + nextSegmentEntryId).whenComplete((___, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey(), bucketId, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes) + .thenApply(bucketSnapshotSegments -> { + if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { + return Collections.emptyList(); + } + + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + bucketSnapshotSegments.get(0); + List indexList = + snapshotSegment.getIndexesList(); + this.setCurrentSegmentEntryId(nextSegmentEntryId); + return indexList; + }); }); } @@ -136,18 +152,34 @@ CompletableFuture> if (nextSegmentEntryId > lastSegmentEntryId) { return CompletableFuture.completedFuture(Collections.emptyList()); } - return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, - lastSegmentEntryId); + return executeWithRetry(() -> { + return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, + lastSegmentEntryId).whenComplete((__, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to get remain bucket snapshot segment, bucketKey: {}.", + dispatcherName, bucketKey(), ex); + } + }); + }, BucketSnapshotPersistenceException.class, MaxRetryTimes); } CompletableFuture asyncDeleteBucketSnapshot() { String bucketKey = bucketKey(); long bucketId = getAndUpdateBucketId(); return removeBucketCursorProperty(bucketKey).thenCompose(__ -> - bucketSnapshotStorage.deleteBucketSnapshot(bucketId)).whenComplete((__, ex) -> { + executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).whenComplete((___, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to delete bucket snapshot. bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey, bucketId, ex); + } + }), + BucketSnapshotPersistenceException.class, MaxRetryTimes)).whenComplete((__, ex) -> { if (ex != null) { - log.warn("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", - bucketId, bucketKey, ex); + log.warn("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", + dispatcherName, bucketId, bucketKey, ex); + } else { + log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", + dispatcherName, bucketId, bucketKey); } }); } @@ -179,8 +211,8 @@ void clear(boolean delete) { try { snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); } catch (Exception e) { - log.warn("Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", getBucketId(), - bucketKey()); + log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName, + getBucketId(), bucketKey()); } } }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index ad457329c427f..40ba8f4c4b593 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -42,9 +42,9 @@ class MutableBucket extends Bucket implements AutoCloseable { private final TripleLongPriorityQueue priorityQueue; - MutableBucket(ManagedCursor cursor, + MutableBucket(String dispatcherName, ManagedCursor cursor, BucketSnapshotStorage bucketSnapshotStorage) { - super(cursor, bucketSnapshotStorage, -1L, -1L); + super(dispatcherName, cursor, bucketSnapshotStorage, -1L, -1L); this.priorityQueue = new TripleLongPriorityQueue(); } @@ -59,6 +59,9 @@ Pair createImmutableBucketAndAsyncPersistent( final long timeStepPerBucketSnapshotSegment, TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId, final long endLedgerId) { + log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName, + startLedgerId, endLedgerId); + if (delayedIndexQueue.isEmpty()) { return null; } @@ -122,11 +125,16 @@ Pair createImmutableBucketAndAsyncPersistent( final int lastSegmentEntryId = segmentMetadataList.size(); - ImmutableBucket bucket = new ImmutableBucket(cursor, bucketSnapshotStorage, startLedgerId, endLedgerId); + ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, bucketSnapshotStorage, + startLedgerId, endLedgerId); bucket.setCurrentSegmentEntryId(1); bucket.setNumberBucketDelayedMessages(numMessages); bucket.setLastSegmentEntryId(lastSegmentEntryId); + // Skip first segment, because it has already been loaded + List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size()); + bucket.setSnapshotSegments(snapshotSegments); + // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); @@ -136,12 +144,6 @@ Pair createImmutableBucketAndAsyncPersistent( CompletableFuture future = asyncSaveBucketSnapshot(bucket, bucketSnapshotMetadata, bucketSnapshotSegments); bucket.setSnapshotCreateFuture(future); - future.whenComplete((__, ex) -> { - if (ex != null) { - //TODO Record create snapshot failed - log.error("Failed to create snapshot: ", ex); - } - }); return result; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 9b2fbda4195da..b106642915587 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -23,8 +23,10 @@ import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -36,6 +38,7 @@ import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.common.util.FutureUtil; @Slf4j public class MockBucketSnapshotStorage implements BucketSnapshotStorage { @@ -53,9 +56,35 @@ public MockBucketSnapshotStorage() { this.maxBucketId = new AtomicLong(); } + public Queue createExceptionQueue = new LinkedList<>(); + public Queue getMetaDataExceptionQueue = new LinkedList<>(); + public Queue getSegmentExceptionQueue = new LinkedList<>(); + public Queue deleteExceptionQueue = new LinkedList<>(); + + + public void injectCreateException(Throwable throwable) { + createExceptionQueue.add(throwable); + } + + public void injectGetMetaDataException(Throwable throwable) { + getMetaDataExceptionQueue.add(throwable); + } + + public void injectGetSegmentException(Throwable throwable) { + getSegmentExceptionQueue.add(throwable); + } + + public void injectDeleteException(Throwable throwable) { + deleteExceptionQueue.add(throwable); + } + @Override public CompletableFuture createBucketSnapshot( SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey) { + Throwable throwable = createExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { long bucketId = maxBucketId.getAndIncrement(); List entries = new ArrayList<>(); @@ -81,6 +110,10 @@ public CompletableFuture createBucketSnapshot( @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { + Throwable throwable = getMetaDataExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(0); SnapshotMetadata snapshotMetadata; @@ -96,6 +129,10 @@ public CompletableFuture getBucketSnapshotMetadata(long bucket @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { + Throwable throwable = getSegmentExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { List snapshotSegments = new ArrayList<>(); long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); @@ -115,6 +152,10 @@ public CompletableFuture> getBucketSnapshotSegment(long bu @Override public CompletableFuture deleteBucketSnapshot(long bucketId) { + Throwable throwable = deleteExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { List remove = this.bucketSnapshots.remove(bucketId); if (remove != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index efb0fa7ab7ba2..499262c1e60b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -46,7 +46,7 @@ public MockManagedCursor(String name) { @Override public String getName() { - return null; + return this.name; } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 920f2cf2b64b3..08e1f78725bf4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -50,6 +50,8 @@ import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; +import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong; +import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -77,6 +79,7 @@ public Object[][] provider(Method method) throws Exception { bucketSnapshotStorage.start(); ManagedCursor cursor = new MockManagedCursor("my_test_cursor"); doReturn(cursor).when(dispatcher).getCursor(); + doReturn(cursor.getName()).when(dispatcher).getName(); final String methodName = method.getName(); return switch (methodName) { @@ -136,7 +139,7 @@ public Object[][] provider(Method method) throws Exception { new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; - case "testMergeSnapshot" -> new Object[][]{{ + case "testMergeSnapshot", "testWithBkException", "testWithCreateFailDowngrade" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10) }}; @@ -255,6 +258,24 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { assertEquals(10, size); + tracker.addMessage(111, 1011, 111 * 10); + + MutableLong delayedMessagesInSnapshot = new MutableLong(); + tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> { + delayedMessagesInSnapshot.add(v.getNumberBucketDelayedMessages()); + }); + + tracker.close(); + + tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10); + + assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue()); + + for (int i = 1; i <= 110; i++) { + tracker.addMessage(i, i, i * 10); + } + clockTime.set(110 * 10); NavigableSet scheduledMessages = tracker.getScheduledMessages(110); @@ -263,4 +284,98 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { assertEquals(position, PositionImpl.get(i, i)); } } + + @Test(dataProvider = "delayedTracker") + public void testWithBkException(BucketDelayedDeliveryTracker tracker) { + MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage; + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectGetMetaDataException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Get entry")); + mockBucketSnapshotStorage.injectDeleteException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Delete entry")); + + assertEquals(1, mockBucketSnapshotStorage.createExceptionQueue.size()); + assertEquals(1, mockBucketSnapshotStorage.getMetaDataExceptionQueue.size()); + assertEquals(1, mockBucketSnapshotStorage.getSegmentExceptionQueue.size()); + assertEquals(1, mockBucketSnapshotStorage.deleteExceptionQueue.size()); + + for (int i = 1; i <= 110; i++) { + tracker.addMessage(i, i, i * 10); + } + + assertEquals(110, tracker.getNumberOfDelayedMessages()); + + int size = tracker.getImmutableBuckets().asMapOfRanges().size(); + + assertEquals(10, size); + + tracker.addMessage(111, 1011, 111 * 10); + + MutableLong delayedMessagesInSnapshot = new MutableLong(); + tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> { + delayedMessagesInSnapshot.add(v.getNumberBucketDelayedMessages()); + }); + + tracker.close(); + + tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10); + + Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue(); + assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue); + + clockTime.set(110 * 10); + + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout1, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout2, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout3, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout4, op: Get entry")); + + assertEquals(tracker.getScheduledMessages(100).size(), 0); + + assertEquals(tracker.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue); + + assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty()); + assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty()); + assertTrue(mockBucketSnapshotStorage.getSegmentExceptionQueue.isEmpty()); + assertTrue(mockBucketSnapshotStorage.deleteExceptionQueue.isEmpty()); + } + + @Test(dataProvider = "delayedTracker") + public void testWithCreateFailDowngrade(BucketDelayedDeliveryTracker tracker) { + MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage; + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + + assertEquals(4, mockBucketSnapshotStorage.createExceptionQueue.size()); + + for (int i = 1; i <= 6; i++) { + tracker.addMessage(i, i, i * 10); + } + + Awaitility.await().untilAsserted(() -> assertEquals(0, tracker.getImmutableBuckets().asMapOfRanges().size())); + + clockTime.set(5 * 10); + + assertEquals(6, tracker.getNumberOfDelayedMessages()); + + NavigableSet scheduledMessages = tracker.getScheduledMessages(5); + for (int i = 1; i <= 5; i++) { + PositionImpl position = scheduledMessages.pollFirst(); + assertEquals(position, PositionImpl.get(i, i)); + } + } } From fec4578e8a5cfeb7e5ad53a0ce47b3e116e94a4d Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 2 Mar 2023 14:29:48 +0800 Subject: [PATCH 038/404] [improve][broker] PIP-192: Add large topic count filter (#19613) --- .../extensions/ExtensibleLoadManagerImpl.java | 2 + .../extensions/data/BrokerLoadData.java | 4 + .../filter/BrokerMaxTopicCountFilter.java | 49 ++++++++ .../reporter/BrokerLoadDataReporter.java | 2 +- .../pulsar/broker/service/PulsarStats.java | 3 +- .../pulsar/broker/stats/BrokerStats.java | 2 + .../ExtensibleLoadManagerImplTest.java | 2 +- .../extensions/data/BrokerLoadDataTest.java | 8 +- .../filter/BrokerFilterTestBase.java | 109 ++++++++++++++++++ .../filter/BrokerMaxTopicCountFilterTest.java | 68 +++++++++++ .../filter/BrokerVersionFilterTest.java | 26 +---- .../reporter/BrokerLoadDataReporterTest.java | 7 +- .../scheduler/TransferShedderTest.java | 2 +- .../LeastResourceUsageWithWeightTest.java | 4 +- 14 files changed, 251 insertions(+), 37 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 59c6674676101..650c12af57391 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -41,6 +41,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; @@ -132,6 +133,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { */ public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline = new ArrayList<>(); + this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); // TODO: Make brokerSelectionStrategy configurable. this.brokerSelectionStrategy = new LeastResourceUsageWithWeight(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java index dbd17152d26e4..8b373bc5954b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java @@ -57,6 +57,7 @@ public class BrokerLoadData { private double msgRateIn; // messages/sec private double msgRateOut; // messages/sec private int bundleCount; + private int topics; // Load data features computed from the above resources. private double maxResourceUsage; // max of resource usages @@ -113,6 +114,7 @@ public void update(final SystemResourceUsage usage, double msgRateIn, double msgRateOut, int bundleCount, + int topics, ServiceConfiguration conf) { updateSystemResourceUsage(usage.cpu, usage.memory, usage.directMemory, usage.bandwidthIn, usage.bandwidthOut); this.msgThroughputIn = msgThroughputIn; @@ -120,6 +122,7 @@ public void update(final SystemResourceUsage usage, this.msgRateIn = msgRateIn; this.msgRateOut = msgRateOut; this.bundleCount = bundleCount; + this.topics = topics; updateFeatures(conf); updatedAt = System.currentTimeMillis(); } @@ -137,6 +140,7 @@ public void update(final BrokerLoadData other) { msgRateIn = other.msgRateIn; msgRateOut = other.msgRateOut; bundleCount = other.bundleCount; + topics = other.topics; weightedMaxEMA = other.weightedMaxEMA; maxResourceUsage = other.maxResourceUsage; updatedAt = other.updatedAt; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java new file mode 100644 index 0000000000000..e3f8faca32468 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import java.util.Map; +import java.util.Optional; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; + +public class BrokerMaxTopicCountFilter implements BrokerFilter { + + public static final String FILTER_NAME = "broker_max_topic_count_filter"; + + @Override + public String name() { + return FILTER_NAME; + } + + @Override + public Map filter(Map brokers, + LoadManagerContext context) throws BrokerFilterException { + int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); + brokers.keySet().removeIf(broker -> { + Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); + long topics = brokerLoadDataOpt.map(BrokerLoadData::getTopics).orElse(0); + // TODO: The broker load data might be delayed, so the max topic check might not accurate. + return topics >= loadBalancerBrokerMaxTopics; + }); + return brokers; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java index cf50f942e11b4..256e52c4554d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java @@ -83,8 +83,8 @@ public BrokerLoadData generateLoadData() { brokerStats.msgRateIn, brokerStats.msgRateOut, brokerStats.bundleCount, + brokerStats.topics, pulsar.getConfiguration()); - } return this.localData; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java index 045bb336d62e2..9cdf9d1dfc68d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java @@ -132,6 +132,7 @@ public synchronized void updateStats( k -> new NamespaceBundleStats()); currentBundleStats.reset(); currentBundleStats.topics = topics.size(); + brokerStats.topics += topics.size(); topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle)); @@ -280,4 +281,4 @@ public void recordConnectionCreateSuccess() { public void recordConnectionCreateFail() { brokerOperabilityMetrics.recordConnectionCreateFail(); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java index d0be71167002d..84d5432fb9e19 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java @@ -21,6 +21,7 @@ public class BrokerStats extends NamespaceStats { public int bundleCount; + public int topics; public BrokerStats(int ratePeriodInSeconds) { super(ratePeriodInSeconds); } @@ -29,5 +30,6 @@ public BrokerStats(int ratePeriodInSeconds) { public void reset() { super.reset(); bundleCount = 0; + topics = 0; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 001aac34a4ba2..d1bf29725d004 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -324,7 +324,7 @@ public void testGetMetrics() throws Exception { usage.setDirectMemory(directMemory); usage.setBandwidthIn(bandwidthIn); usage.setBandwidthOut(bandwidthOut); - loadData.update(usage, 1, 2, 3, 4, 5, conf); + loadData.update(usage, 1, 2, 3, 4, 5, 6, conf); brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); } { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java index c85fa4ce9d2cd..5ba7629dd1132 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java @@ -59,7 +59,7 @@ public void testUpdateBySystemResourceUsage() { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - data.update(usage1, 1, 2, 3, 4, 5, conf); + data.update(usage1, 1, 2, 3, 4, 5, 6, conf); assertEquals(data.getCpu(), cpu); assertEquals(data.getMemory(), memory); @@ -71,6 +71,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getMsgRateIn(), 3.0); assertEquals(data.getMsgRateOut(), 4.0); assertEquals(data.getBundleCount(), 5); + assertEquals(data.getTopics(), 6); assertEquals(data.getMaxResourceUsage(), 0.04); // skips memory usage assertEquals(data.getWeightedMaxEMA(), 2); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); @@ -87,7 +88,7 @@ public void testUpdateBySystemResourceUsage() { usage2.setDirectMemory(directMemory); usage2.setBandwidthIn(bandwidthIn); usage2.setBandwidthOut(bandwidthOut); - data.update(usage2, 5, 6, 7, 8, 9, conf); + data.update(usage2, 5, 6, 7, 8, 9, 10, conf); assertEquals(data.getCpu(), cpu); assertEquals(data.getMemory(), memory); @@ -99,6 +100,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getMsgRateIn(), 7.0); assertEquals(data.getMsgRateOut(), 8.0); assertEquals(data.getBundleCount(), 9); + assertEquals(data.getTopics(), 10); assertEquals(data.getMaxResourceUsage(), 3.0); assertEquals(data.getWeightedMaxEMA(), 1.875); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); @@ -137,7 +139,7 @@ public void testUpdateByBrokerLoadData() { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - other.update(usage1, 1, 2, 3, 4, 5, conf); + other.update(usage1, 1, 2, 3, 4, 5, 6, conf); data.update(other); assertEquals(data, other); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java new file mode 100644 index 0000000000000..3de957c3b1a3a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; + +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; + +public class BrokerFilterTestBase { + + public LoadManagerContext getContext() { + LoadManagerContext mockContext = mock(LoadManagerContext.class); + ServiceConfiguration configuration = new ServiceConfiguration(); + var brokerLoadDataStore = new LoadDataStore() { + Map map = new HashMap<>(); + @Override + public void close() throws IOException { + + } + + @Override + public CompletableFuture pushAsync(String key, BrokerLoadData loadData) { + map.put(key, loadData); + return null; + } + + @Override + public CompletableFuture removeAsync(String key) { + return null; + } + + @Override + public Optional get(String key) { + var val = map.get(key); + if (val == null) { + return Optional.empty(); + } + return Optional.of(val); + } + + @Override + public void forEach(BiConsumer action) { + + } + + @Override + public Set> entrySet() { + return map.entrySet(); + } + + @Override + public int size() { + return map.size(); + } + }; + configuration.setPreferLaterVersions(true); + doReturn(configuration).when(mockContext).brokerConfiguration(); + doReturn(brokerLoadDataStore).when(mockContext).brokerLoadDataStore(); + return mockContext; + } + + public BrokerLookupData getLookupData() { + return getLookupData("3.0.0"); + } + + public BrokerLookupData getLookupData(String version) { + String webServiceUrl = "http://localhost:8080"; + String webServiceUrlTls = "https://localhoss:8081"; + String pulsarServiceUrl = "pulsar://localhost:6650"; + String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, true, true, version); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java new file mode 100644 index 0000000000000..4c3255341b778 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import org.apache.commons.lang.reflect.FieldUtils; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +/** + * Unit test for {@link BrokerMaxTopicCountFilter}. + */ +@Test(groups = "broker") +public class BrokerMaxTopicCountFilterTest extends BrokerFilterTestBase { + + @Test + public void test() throws IllegalAccessException, BrokerFilterException { + LoadManagerContext context = getContext(); + LoadDataStore store = context.brokerLoadDataStore(); + BrokerLoadData maxTopicLoadData = new BrokerLoadData(); + FieldUtils.writeDeclaredField(maxTopicLoadData, "topics", + context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(), true); + BrokerLoadData exceedMaxTopicLoadData = new BrokerLoadData(); + FieldUtils.writeDeclaredField(exceedMaxTopicLoadData, "topics", + context.brokerConfiguration().getLoadBalancerBrokerMaxTopics() * 2, true); + store.pushAsync("broker1", maxTopicLoadData); + store.pushAsync("broker2", new BrokerLoadData()); + store.pushAsync("broker3", exceedMaxTopicLoadData); + + BrokerMaxTopicCountFilter filter = new BrokerMaxTopicCountFilter(); + Map originalBrokers = Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData(), + "broker4", getLookupData() + ); + Map result = filter.filter(new HashMap<>(originalBrokers), context); + assertEquals(result, Map.of( + "broker2", getLookupData(), + "broker4", getLookupData() + )); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index 1fcc3836a6fac..d36c79d60ed4e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -30,14 +29,13 @@ import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; -import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.testng.annotations.Test; /** * Unit test for {@link BrokerVersionFilter}. */ @Test(groups = "broker") -public class BrokerVersionFilterTest { +public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test @@ -115,26 +113,4 @@ public void testInvalidVersionString() throws BrokerFilterException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); } - - public LoadManagerContext getContext() { - LoadManagerContext mockContext = mock(LoadManagerContext.class); - ServiceConfiguration configuration = new ServiceConfiguration(); - configuration.setPreferLaterVersions(true); - doReturn(configuration).when(mockContext).brokerConfiguration(); - return mockContext; - } - - public BrokerLookupData getLookupData(String version) { - String webServiceUrl = "http://localhost:8080"; - String webServiceUrlTls = "https://localhoss:8081"; - String pulsarServiceUrl = "pulsar://localhost:6650"; - String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; - Map advertisedListeners = new HashMap<>(); - Map protocols = new HashMap<>(){{ - put("kafka", "9092"); - }}; - return new BrokerLookupData( - webServiceUrl, webServiceUrlTls, pulsarServiceUrl, - pulsarServiceUrlTls, advertisedListeners, protocols, true, true, version); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java index 3e4dc4cb1c07b..ee7e708667a32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java @@ -65,6 +65,7 @@ void setup() { doReturn(Executors.newSingleThreadScheduledExecutor()).when(pulsar).getLoadManagerExecutor(); doReturn(pulsarStats).when(brokerService).getPulsarStats(); brokerStats = new BrokerStats(0); + brokerStats.topics = 6; brokerStats.bundleCount = 5; brokerStats.msgRateIn = 3; brokerStats.msgRateOut = 4; @@ -88,7 +89,7 @@ public void testGenerate() throws IllegalAccessException { doReturn(0l).when(pulsarStats).getUpdatedAt(); var target = new BrokerLoadDataReporter(pulsar, "", store); var expected = new BrokerLoadData(); - expected.update(usage, 1, 2, 3, 4, 5, config); + expected.update(usage, 1, 2, 3, 4, 5, 6, config); FieldUtils.writeDeclaredField(expected, "updatedAt", 0l, true); var actual = target.generateLoadData(); FieldUtils.writeDeclaredField(actual, "updatedAt", 0l, true); @@ -103,7 +104,7 @@ public void testReport() throws IllegalAccessException { var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); localData.setReportedAt(System.currentTimeMillis()); var lastData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "lastData", true); - lastData.update(usage, 1, 2, 3, 4, 5, config); + lastData.update(usage, 1, 2, 3, 4, 5, 6, config); target.reportAsync(false); verify(store, times(0)).pushAsync(any(), any()); @@ -117,7 +118,7 @@ public void testReport() throws IllegalAccessException { target.reportAsync(false); verify(store, times(2)).pushAsync(eq("broker-1"), any()); - lastData.update(usage, 10000, 2, 3, 4, 5, config); + lastData.update(usage, 10000, 2, 3, 4, 5, 6, config); target.reportAsync(false); verify(store, times(3)).pushAsync(eq("broker-1"), any()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index bdf5f846267e6..709a1113f35e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -121,7 +121,7 @@ public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - loadData.update(usage1, 1,2,3,4,5, + loadData.update(usage1, 1,2,3,4,5,6, ctx.brokerConfiguration()); return loadData; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java index da0866c689746..ebc2424cada19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java @@ -168,7 +168,7 @@ public void testNoLoadDataBrokers() { private BrokerLoadData createBrokerData(LoadManagerContext ctx, double usage, double limit) { var brokerLoadData = new BrokerLoadData(); SystemResourceUsage usages = createUsage(usage, limit); - brokerLoadData.update(usages, 1, 1, 1, 1, 1, + brokerLoadData.update(usages, 1, 1, 1, 1, 1, 1, ctx.brokerConfiguration()); return brokerLoadData; } @@ -185,7 +185,7 @@ private SystemResourceUsage createUsage(double usage, double limit) { private void updateLoad(LoadManagerContext ctx, String broker, double usage) { ctx.brokerLoadDataStore().get(broker).get().update(createUsage(usage, 100.0), - 1, 1, 1, 1, 1, ctx.brokerConfiguration()); + 1, 1, 1, 1, 1, 1, ctx.brokerConfiguration()); } public static LoadManagerContext getContext() { From 0e6ef109322058e7834ca23d968f66f5632cf2b1 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 2 Mar 2023 19:11:43 -0800 Subject: [PATCH 039/404] [improve][broker] PIP-192 Switched Assigning and Releasing state order in the transfer protocol (#19683) --- .../extensions/channel/ServiceUnitState.java | 9 +- .../channel/ServiceUnitStateChannelImpl.java | 188 ++++++++++-------- .../ServiceUnitStateCompactionStrategy.java | 28 ++- .../channel/ServiceUnitStateData.java | 16 +- .../channel/ServiceUnitStateChannelTest.java | 12 +- ...erviceUnitStateCompactionStrategyTest.java | 43 ++-- .../channel/ServiceUnitStateDataTest.java | 13 +- .../channel/ServiceUnitStateTest.java | 19 +- .../ServiceUnitStateCompactionTest.java | 14 +- 9 files changed, 191 insertions(+), 151 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java index 92fef8f65992a..42ef55593ae1a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java @@ -47,9 +47,9 @@ public enum ServiceUnitState { // when the topic is compacted in the middle of assign, transfer or split. Init, Set.of(Free, Owned, Assigning, Releasing, Splitting, Deleted), Free, Set.of(Assigning, Init), - Owned, Set.of(Assigning, Splitting, Releasing), - Assigning, Set.of(Owned, Releasing), - Releasing, Set.of(Owned, Free), + Owned, Set.of(Splitting, Releasing), + Assigning, Set.of(Owned), + Releasing, Set.of(Assigning, Free), Splitting, Set.of(Deleted), Deleted, Set.of(Init) ); @@ -67,4 +67,7 @@ public static boolean isInFlightState(ServiceUnitState state) { return inFlightStates.contains(state); } + public static boolean isActiveState(ServiceUnitState state) { + return inFlightStates.contains(state) || state == Owned; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 4819a54fcad73..cddaf92d22771 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -28,6 +28,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isActiveState; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isInFlightState; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Closed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Constructed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.LeaderElectionServiceStarted; @@ -430,8 +432,11 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { ServiceUnitState state = state(data); ownerLookUpCounters.get(state).incrementAndGet(); switch (state) { - case Owned, Splitting -> { - return CompletableFuture.completedFuture(Optional.of(data.broker())); + case Owned -> { + return CompletableFuture.completedFuture(Optional.of(data.dstBroker())); + } + case Splitting -> { + return CompletableFuture.completedFuture(Optional.of(data.sourceBroker())); } case Assigning, Releasing -> { return deferGetOwnerRequest(serviceUnit).thenApply( @@ -490,16 +495,15 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { EventType eventType = Unload; eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = unload.serviceUnit(); - CompletableFuture future; + ServiceUnitStateData next; if (isTransferCommand(unload)) { - future = pubAsync(serviceUnit, new ServiceUnitStateData( - Assigning, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit))); + next = new ServiceUnitStateData( + Releasing, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit)); } else { - future = pubAsync(serviceUnit, new ServiceUnitStateData( - Releasing, unload.sourceBroker(), getNextVersionId(serviceUnit))); + next = new ServiceUnitStateData( + Releasing, null, unload.sourceBroker(), getNextVersionId(serviceUnit)); } - - return future.whenComplete((__, ex) -> { + return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); } @@ -515,7 +519,7 @@ public CompletableFuture publishSplitEventAsync(Split split) { eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = split.serviceUnit(); ServiceUnitStateData next = - new ServiceUnitStateData(Splitting, split.sourceBroker(), getNextVersionId(serviceUnit)); + new ServiceUnitStateData(Splitting, null, split.sourceBroker(), getNextVersionId(serviceUnit)); return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); @@ -553,7 +557,8 @@ private static boolean isTransferCommand(ServiceUnitStateData data) { if (data == null) { return false; } - return StringUtils.isNotEmpty(data.sourceBroker()); + return StringUtils.isNotEmpty(data.dstBroker()) + && StringUtils.isNotEmpty(data.sourceBroker()); } private static boolean isTransferCommand(Unload data) { @@ -612,45 +617,41 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { - getOwnerRequest.complete(data.broker()); + getOwnerRequest.complete(data.dstBroker()); } - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); } } private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.dstBroker())) { ServiceUnitStateData next = new ServiceUnitStateData( - isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker(), - getNextVersionId(data)); + Owned, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); pubAsync(serviceUnit, next) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTransferCommand(data)) { - if (isTargetBroker(data.sourceBroker())) { - ServiceUnitStateData next = - new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker(), getNextVersionId(data)); + if (isTargetBroker(data.sourceBroker())) { + ServiceUnitStateData next; + if (isTransferCommand(data)) { + next = new ServiceUnitStateData( + Assigning, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); // TODO: when close, pass message to clients to connect to the new broker - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) - .whenComplete((__, e) -> log(e, serviceUnit, data, next)); - } - } else { - if (isTargetBroker(data.broker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker(), getNextVersionId(data)); - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) - .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + } else { + next = new ServiceUnitStateData( + Free, null, data.sourceBroker(), getNextVersionId(data)); } + closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)) + .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.sourceBroker())) { splitServiceUnit(serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, null)); } @@ -661,7 +662,7 @@ private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } } @@ -671,7 +672,7 @@ private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); } - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } } @@ -795,7 +796,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); return; } - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), VERSION_ID_INIT); + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.sourceBroker(), VERSION_ID_INIT); NamespaceBundles targetNsBundle = splitBundlesPair.getLeft(); List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight()); List successPublishedBundles = @@ -833,7 +834,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.thenAccept(r -> { // Delete the old bundle - pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker(), getNextVersionId(data))) + pubAsync(serviceUnit, new ServiceUnitStateData( + Deleted, null, data.sourceBroker(), getNextVersionId(data))) .thenRun(() -> { // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); @@ -983,18 +985,19 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); var availableBrokers = new HashSet<>(brokerRegistry.getAvailableBrokersAsync() .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); + for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); var serviceUnit = etr.getKey(); var state = state(stateData); - if (StringUtils.equals(broker, stateData.broker())) { - if (ServiceUnitState.isInFlightState(state) || state == Owned) { + if (StringUtils.equals(broker, stateData.dstBroker())) { + if (isActiveState(state)) { overrideOwnership(serviceUnit, stateData, availableBrokers); orphanServiceUnitCleanupCnt++; } } else if (StringUtils.equals(broker, stateData.sourceBroker())) { - if (ServiceUnitState.isInFlightState(state)) { + if (isInFlightState(state)) { overrideOwnership(serviceUnit, stateData, availableBrokers); orphanServiceUnitCleanupCnt++; } @@ -1026,39 +1029,57 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce cleanupJobs.remove(broker); } - private Optional getOverrideStateData(String serviceUnit, ServiceUnitStateData orphanData, - Set availableBrokers, - LoadManagerContext context) { + private Optional getRollForwardStateData( + Set availableBrokers, LoadManagerContext context, long nextVersionId) { + Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); + if (selectedBroker.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); + } + + private Optional getOverrideInFlightStateData( + String serviceUnit, ServiceUnitStateData orphanData, + Set availableBrokers, + LoadManagerContext context) { long nextVersionId = getNextVersionId(orphanData); - if (isTransferCommand(orphanData)) { - // rollback to the src - return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); - } else if (orphanData.state() == Assigning) { // assign - // roll-forward to another broker - Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); - if (selectedBroker.isEmpty()) { - return Optional.empty(); + var state = orphanData.state(); + switch (state) { + case Assigning: { + return getRollForwardStateData(availableBrokers, context, nextVersionId); + } + case Splitting, Releasing: { + if (availableBrokers.contains(orphanData.sourceBroker())) { + // rollback to the src + return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); + } else { + return getRollForwardStateData(availableBrokers, context, nextVersionId); + } + } + default: { + var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", + serviceUnit, orphanData); + log.error(msg); + throw new IllegalStateException(msg); } - return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); - } else if (orphanData.state() == Splitting || orphanData.state() == Releasing) { - // rollback to the target broker for split and unload - return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true, nextVersionId)); - } else { - var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", - serviceUnit, orphanData); - log.error(msg); - throw new IllegalStateException(msg); } } @VisibleForTesting protected void monitorOwnerships(List brokers) { if (!isChannelOwner()) { - log.warn("This broker is not the leader now. Skipping ownership monitor"); + log.warn("This broker is not the leader now. Skipping ownership monitor."); return; } + if (brokers == null || brokers.size() == 0) { - log.error("no active brokers found. Skipping the ownership monitor run."); + log.error("no active brokers found. Skipping ownership monitor."); + return; + } + + var metadataState = getMetadataState(); + if (metadataState != Stable) { + log.warn("metadata state:{} is not Stable. Skipping ownership monitor.", metadataState); return; } @@ -1071,34 +1092,35 @@ protected void monitorOwnerships(List brokers) { int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); long now = System.currentTimeMillis(); - for (Map.Entry etr : tableview.entrySet()) { + for (var etr : tableview.entrySet()) { String serviceUnit = etr.getKey(); ServiceUnitStateData stateData = etr.getValue(); - String broker = stateData.broker(); + String dstBroker = stateData.dstBroker(); + String srcBroker = stateData.sourceBroker(); var state = stateData.state(); - if (!activeBrokers.contains(broker)) { - inactiveBrokers.add(stateData.broker()); - } else if (state != Owned - && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - if (state == Deleted || state == Free) { - if (now - stateData.timestamp() - > semiTerminalStateWaitingTimeInMillis) { - log.info("Found semi-terminal states to tombstone" - + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " - + "cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); - } - }); - serviceUnitTombstoneCleanupCnt++; - } - } else { + + if (isActiveState(state)) { + if (StringUtils.isNotBlank(srcBroker) && !activeBrokers.contains(srcBroker)) { + inactiveBrokers.add(srcBroker); + } else if (StringUtils.isNotBlank(dstBroker) && !activeBrokers.contains(dstBroker)) { + inactiveBrokers.add(dstBroker); + } else if (isInFlightState(state) + && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { log.warn("Found orphan serviceUnit:{}, stateData:{}", serviceUnit, stateData); orphanServiceUnits.put(serviceUnit, stateData); } + } else if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) { + log.info("Found semi-terminal states to tombstone" + + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); + tombstoneAsync(serviceUnit).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " + + "cleanupErrorCnt:{}.", + serviceUnit, stateData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); + serviceUnitTombstoneCleanupCnt++; } } @@ -1112,7 +1134,7 @@ protected void monitorOwnerships(List brokers) { for (var etr : orphanServiceUnits.entrySet()) { var orphanServiceUnit = etr.getKey(); var orphanData = etr.getValue(); - var overrideData = getOverrideStateData( + var overrideData = getOverrideInFlightStateData( orphanServiceUnit, orphanData, activeBrokers, context); if (overrideData.isPresent()) { pubAsync(orphanServiceUnit, overrideData.get()).whenComplete((__, e) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index 8af0f0c027da4..ceb3ea3e9cb6c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -71,22 +71,21 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to switch (prevState) { case Owned: switch (state) { - case Assigning: - return invalidTransfer(from, to); case Splitting: + return isNotBlank(to.dstBroker()) + || !from.dstBroker().equals(to.sourceBroker()); case Releasing: - return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + return invalidUnload(from, to); } case Assigning: switch (state) { - case Releasing: - return isBlank(to.sourceBroker()) || notEquals(from, to); case Owned: - return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + return notEquals(from, to); } case Releasing: switch (state) { - case Owned: + case Assigning: + return isBlank(to.dstBroker()) || notEquals(from, to); case Free: return notEquals(from, to); } @@ -98,24 +97,21 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to case Free: switch (state) { case Assigning: - return isNotBlank(to.sourceBroker()); + return isNotBlank(to.sourceBroker()) || isBlank(to.dstBroker()); } } } return false; } - private boolean targetNotEquals(ServiceUnitStateData from, ServiceUnitStateData to) { - return !from.broker().equals(to.broker()); - } - private boolean notEquals(ServiceUnitStateData from, ServiceUnitStateData to) { - return !from.broker().equals(to.broker()) + return !StringUtils.equals(from.dstBroker(), to.dstBroker()) || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); } - private boolean invalidTransfer(ServiceUnitStateData from, ServiceUnitStateData to) { - return !from.broker().equals(to.sourceBroker()) - || from.broker().equals(to.broker()); + private boolean invalidUnload(ServiceUnitStateData from, ServiceUnitStateData to) { + return isBlank(to.sourceBroker()) + || !from.dstBroker().equals(to.sourceBroker()) + || from.dstBroker().equals(to.dstBroker()); } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index ef25acff10a4b..c26dce83a4434 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -27,25 +27,25 @@ */ public record ServiceUnitStateData( - ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp, long versionId) { + ServiceUnitState state, String dstBroker, String sourceBroker, boolean force, long timestamp, long versionId) { public ServiceUnitStateData { Objects.requireNonNull(state); - if (StringUtils.isBlank(broker)) { + if (StringUtils.isBlank(dstBroker) && StringUtils.isBlank(sourceBroker)) { throw new IllegalArgumentException("Empty broker"); } } - public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker, long versionId) { - this(state, broker, sourceBroker, false, System.currentTimeMillis(), versionId); + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) { + this(state, dstBroker, sourceBroker, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker, long versionId) { - this(state, broker, null, false, System.currentTimeMillis(), versionId); + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) { + this(state, dstBroker, null, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force, long versionId) { - this(state, broker, null, force, System.currentTimeMillis(), versionId); + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, boolean force, long versionId) { + this(state, dstBroker, null, force, System.currentTimeMillis(), versionId); } public static ServiceUnitState state(ServiceUnitStateData data) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 6aa8fe387605b..64a5f63196bea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -152,13 +152,15 @@ protected void setup() throws Exception { } @BeforeMethod - protected void initTableViews() throws Exception { + protected void initChannels() throws Exception { cleanTableView(channel1, bundle); cleanTableView(channel2, bundle); cleanOwnershipMonitorCounters(channel1); cleanOwnershipMonitorCounters(channel2); cleanOpsCounters(channel1); cleanOpsCounters(channel2); + cleanMetadataState(channel1); + cleanMetadataState(channel2); } @@ -491,6 +493,7 @@ public void transferTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests2.size()); // recovered, check the monitor update state : Assigned -> Owned + doReturn(Optional.of(lookupServiceAddress1)).when(brokerSelector).select(any(), any(), any()); FieldUtils.writeDeclaredField(channel2, "producer", producer, true); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 1 , true); @@ -937,7 +940,7 @@ public void ownerLookupCountTests() throws IllegalAccessException { channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1", 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, "b1", 1)); channel1.getOwnerAsync(bundle); overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1", 1)); @@ -1363,6 +1366,11 @@ private void cleanOwnershipMonitorCounters(ServiceUnitStateChannel channel) thro FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupCancelledCnt", 0, true); } + private void cleanMetadataState(ServiceUnitStateChannel channel) throws IllegalAccessException { + channel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(channel, "lastMetadataSessionEventTimestamp", 0L, true); + } + private static long getCleanupMetric(ServiceUnitStateChannel channel, String metric) throws IllegalAccessException { Object var = FieldUtils.readDeclaredField(channel, metric, true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 0cd05d8bd7559..64964826af652 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -71,19 +71,19 @@ public void testVersionId(){ assertFalse(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, 10), - new ServiceUnitStateData(Assigning, "broker2", dst, 11))); + new ServiceUnitStateData(Releasing, "broker2", dst, 11))); assertFalse(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE), - new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 1))); + new ServiceUnitStateData(Releasing, "broker2", dst, Long.MAX_VALUE + 1))); assertFalse(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE + 1), - new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 2))); + new ServiceUnitStateData(Releasing, "broker2", dst, Long.MAX_VALUE + 2))); assertTrue(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, 10), - new ServiceUnitStateData(Assigning, "broker2", dst, 5))); + new ServiceUnitStateData(Releasing, "broker2", dst, 5))); } @@ -133,41 +133,42 @@ public void testTransitionsAndBrokers() { assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data2(Owned, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, src, dst))); assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Owned, src, dst))); assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data2(Releasing, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data2(Releasing, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, src, dst))); assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Splitting, dst))); assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Deleted, dst))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Init))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Free))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Assigning, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, "dst1"))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Assigning))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Owned))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Releasing, dst))); assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Releasing, src, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Releasing, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Releasing, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, null, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, src, null))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst, null))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "src1", dst), data2(Releasing, "src2", dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, src, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Splitting, src, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Splitting, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Splitting, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Splitting, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Splitting, dst, null))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Splitting, dst, null))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Deleted, dst))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Init))); assertFalse(strategy.shouldKeepLeft(data(Releasing), data2(Free))); assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Free, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Free, "src2", dst))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Owned, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Owned, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data2(Owned, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Assigning, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data2(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Owned))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Releasing))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Splitting))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Deleted, dst))); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java index a48e2a4db8b37..9bfa8ad3ac5f7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java @@ -35,16 +35,15 @@ public class ServiceUnitStateDataTest { public void testConstructors() throws InterruptedException { ServiceUnitStateData data1 = new ServiceUnitStateData(Owned, "A", 1); assertEquals(data1.state(), Owned); - assertEquals(data1.broker(), "A"); + assertEquals(data1.dstBroker(), "A"); assertNull(data1.sourceBroker()); assertThat(data1.timestamp()).isGreaterThan(0); - ; Thread.sleep(10); ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B", 1); assertEquals(data2.state(), Assigning); - assertEquals(data2.broker(), "A"); + assertEquals(data2.dstBroker(), "A"); assertEquals(data2.sourceBroker(), "B"); assertThat(data2.timestamp()).isGreaterThan(data1.timestamp()); } @@ -55,13 +54,13 @@ public void testNullState() { } @Test(expectedExceptions = IllegalArgumentException.class) - public void testNullBroker() { - new ServiceUnitStateData(Owned, null, 1); + public void testNullBrokers() { + new ServiceUnitStateData(Owned, null, null, 1); } @Test(expectedExceptions = IllegalArgumentException.class) - public void testEmptyBroker() { - new ServiceUnitStateData(Owned, "", 1); + public void testEmptyBrokers() { + new ServiceUnitStateData(Owned, "", "", 1); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java index f5f1fe7bc575f..620266aee46a1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java @@ -42,6 +42,17 @@ public void testInFlights() { assertFalse(ServiceUnitState.isInFlightState(Deleted)); } + @Test + public void testActive() { + assertFalse(ServiceUnitState.isActiveState(Init)); + assertFalse(ServiceUnitState.isActiveState(Free)); + assertTrue(ServiceUnitState.isActiveState(Owned)); + assertTrue(ServiceUnitState.isActiveState(Assigning)); + assertTrue(ServiceUnitState.isActiveState(Releasing)); + assertTrue(ServiceUnitState.isActiveState(Splitting)); + assertFalse(ServiceUnitState.isActiveState(Deleted)); + } + @Test public void testTransitions() { @@ -65,13 +76,13 @@ public void testTransitions() { assertFalse(ServiceUnitState.isValidTransition(Assigning, Free)); assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning)); assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Assigning, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Releasing)); assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting)); assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted)); assertFalse(ServiceUnitState.isValidTransition(Owned, Init)); assertFalse(ServiceUnitState.isValidTransition(Owned, Free)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Assigning)); assertFalse(ServiceUnitState.isValidTransition(Owned, Owned)); assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing)); assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting)); @@ -79,8 +90,8 @@ public void testTransitions() { assertFalse(ServiceUnitState.isValidTransition(Releasing, Init)); assertTrue(ServiceUnitState.isValidTransition(Releasing, Free)); - assertFalse(ServiceUnitState.isValidTransition(Releasing, Assigning)); - assertTrue(ServiceUnitState.isValidTransition(Releasing, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Owned)); assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing)); assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting)); assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 543b7c629ac5f..02812898dc49a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -570,12 +570,12 @@ public void testSlowTableviewAfterCompaction() throws Exception { long versionId = 1; producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src, versionId++)).send(); for (int i = 0; i < 3; i++) { - var assignedStateData = new ServiceUnitStateData(Assigning, dst, src, versionId++); - producer.newMessage().key(bundle).value(assignedStateData).send(); - producer.newMessage().key(bundle).value(assignedStateData).send(); var releasedStateData = new ServiceUnitStateData(Releasing, dst, src, versionId++); producer.newMessage().key(bundle).value(releasedStateData).send(); producer.newMessage().key(bundle).value(releasedStateData).send(); + var assignedStateData = new ServiceUnitStateData(Assigning, dst, src, versionId++); + producer.newMessage().key(bundle).value(assignedStateData).send(); + producer.newMessage().key(bundle).value(assignedStateData).send(); var ownedStateData = new ServiceUnitStateData(Owned, dst, src, versionId++); producer.newMessage().key(bundle).value(ownedStateData).send(); producer.newMessage().key(bundle).value(ownedStateData).send(); @@ -726,7 +726,7 @@ public void testWholeBatchCompactedOut() throws Exception { .subscriptionName("sub1").readCompacted(true).subscribe()) { Message message = consumer.receive(); Assert.assertEquals(message.getKey(), "key1"); - Assert.assertEquals(new String(message.getValue().broker()), "my-message-4"); + Assert.assertEquals(new String(message.getValue().dstBroker()), "my-message-4"); } } @@ -860,7 +860,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() Message m1 = consumer.receive(); assertNotNull(m1); assertEquals(m1.getKey(), key); - assertEquals(m1.getValue().broker(), "19"); + assertEquals(m1.getValue().dstBroker(), "19"); Message none = consumer.receive(2, TimeUnit.SECONDS); assertNull(none); } @@ -908,7 +908,7 @@ public void testReadUnCompacted() Message received = consumer.receive(); assertNotNull(received); assertEquals(received.getKey(), key); - assertEquals(received.getValue().broker(), i + 9 + ""); + assertEquals(received.getValue().dstBroker(), i + 9 + ""); consumer.acknowledge(received); } Message none = consumer.receive(2, TimeUnit.SECONDS); @@ -946,7 +946,7 @@ public void testReadUnCompacted() Message received = consumer.receive(); assertNotNull(received); assertEquals(received.getKey(), key); - assertEquals(received.getValue().broker(), i + 20 + ""); + assertEquals(received.getValue().dstBroker(), i + 20 + ""); consumer.acknowledge(received); } Message none = consumer.receive(2, TimeUnit.SECONDS); From d17bdf85d184c37cfcc16cfcb8df6f2e198e6fec Mon Sep 17 00:00:00 2001 From: YANGLiiN Date: Fri, 3 Mar 2023 19:29:51 +0800 Subject: [PATCH 040/404] [fix][admin] fix create-subscription command --subscription description (#19591) --- .../java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java index 5dbb3935ce98a..6e59be39c018b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java @@ -502,7 +502,7 @@ private class CreateSubscription extends CliCommand { private java.util.List params; @Parameter(names = { "-s", - "--subscription" }, description = "Subscription to reset position on", required = true) + "--subscription" }, description = "Subscription name", required = true) private String subscriptionName; @Parameter(names = { "--messageId", From bce80d9b525d1b21f8440185e7529d492db420eb Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Sat, 4 Mar 2023 00:22:30 +0800 Subject: [PATCH 041/404] [fix][doc] correct description of REST API doc (namespace) (#19580) --- .../main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 12ff229c2f0ed..55766c173f71f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -2569,7 +2569,7 @@ public void getProperty( @DELETE @Path("/{tenant}/{namespace}/property/{key}") - @ApiOperation(value = "Get property value for a given key on a namespace.") + @ApiOperation(value = "Remove property value for a given key on a namespace.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void removeProperty( From e75def80ff2f366919b55cfd42f1dfab3f719b88 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Sat, 4 Mar 2023 08:46:50 -0800 Subject: [PATCH 042/404] [fix][cli] Fix Pulsar admin tool is ignoring tls-trust-cert path arg (#19696) --- .../internal/PulsarAdminBuilderImpl.java | 2 ++ .../pulsar/admin/cli/PulsarAdminSupplier.java | 5 ++- .../pulsar/admin/cli/PulsarAdminTool.java | 12 +++++-- .../apache/pulsar/admin/cli/TestRunMain.java | 35 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 3e7ee472e464b..009fa67fbaa29 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -33,6 +34,7 @@ public class PulsarAdminBuilderImpl implements PulsarAdminBuilder { + @Getter protected ClientConfigurationData conf; private ClassLoader clientBuilderClassLoader = null; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java index 6aa8d2b9c610a..764dc9de5dfdd 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java @@ -53,7 +53,7 @@ static RootParamsKey fromRootParams(PulsarAdminTool.RootParams params) { } } - private final PulsarAdminBuilder adminBuilder; + protected final PulsarAdminBuilder adminBuilder; private RootParamsKey currentParamsKey; private PulsarAdmin admin; @@ -103,6 +103,9 @@ private static void applyRootParamsToAdminBuilder(PulsarAdminBuilder adminBuilde if (isNotBlank(rootParams.tlsProvider)) { adminBuilder.sslProvider(rootParams.tlsProvider); } + if (isNotBlank(rootParams.tlsTrustCertsFilePath)) { + adminBuilder.tlsTrustCertsFilePath(rootParams.tlsTrustCertsFilePath); + } } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index 7a7a4c4b44449..ca0a8a055cfc9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -45,7 +45,7 @@ public class PulsarAdminTool { - private static boolean allowSystemExit = true; + protected static boolean allowSystemExit = true; private static int lastExitCode = Integer.MIN_VALUE; @@ -54,7 +54,7 @@ public class PulsarAdminTool { protected JCommander jcommander; protected RootParams rootParams; private final Properties properties; - private PulsarAdminSupplier pulsarAdminSupplier; + protected PulsarAdminSupplier pulsarAdminSupplier; @Getter public static class RootParams { @@ -277,11 +277,16 @@ protected boolean run(String[] args) { } public static void main(String[] args) throws Exception { + execute(args); + } + + @VisibleForTesting + public static PulsarAdminTool execute(String[] args) throws Exception { lastExitCode = 0; if (args.length == 0) { System.out.println("Usage: pulsar-admin CONF_FILE_PATH [options] [command] [command options]"); exit(0); - return; + return null; } String configFile = args[0]; Properties properties = new Properties(); @@ -299,6 +304,7 @@ public static void main(String[] args) throws Exception { } else { exit(1); } + return tool; } private static void exit(int code) { diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java index 3ac7abf0efc41..c3dbd1cdc7c85 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java @@ -19,9 +19,15 @@ package org.apache.pulsar.admin.cli; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.util.Properties; +import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl; import org.testng.annotations.Test; public class TestRunMain { @@ -75,4 +81,33 @@ public void testRunWithTlsProviderFlagWithConfigFile() throws Exception { "tenants"}); assertEquals(pulsarAdminTool.rootParams.tlsProvider, "OPENSSL"); } + + @Test + public void testMainArgs() throws Exception { + String tlsTrustCertsFilePathInFile = "ca-file.cert"; + String tlsTrustCertsFilePathInArg = "ca-arg.cert"; + File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); + if (testConfigFile.exists()) { + testConfigFile.delete(); + } + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile))); + printWriter.println("tlsTrustCertsFilePath=" + tlsTrustCertsFilePathInFile); + printWriter.println("tlsAllowInsecureConnection=" + false); + printWriter.println("tlsEnableHostnameVerification=" + false); + + printWriter.close(); + testConfigFile.deleteOnExit(); + + String argStrTemp = "%s %s --admin-url https://url:4443 " + "topics stats persistent://prop/cluster/ns/t1"; + boolean prevValue = PulsarAdminTool.allowSystemExit; + PulsarAdminTool.allowSystemExit = false; + + String argStr = argStr = argStrTemp.format(argStrTemp, testConfigFile.getAbsolutePath(), + "--tls-trust-cert-path " + tlsTrustCertsFilePathInArg); + PulsarAdminTool tool = PulsarAdminTool.execute(argStr.split(" ")); + assertNotNull(tool); + PulsarAdminBuilderImpl builder = (PulsarAdminBuilderImpl) tool.pulsarAdminSupplier.adminBuilder; + assertEquals(builder.getConf().getTlsTrustCertsFilePath(), tlsTrustCertsFilePathInArg); + PulsarAdminTool.allowSystemExit = prevValue; + } } From e0fe818633ab170d27e8a120ff8c9d5bc2f01a9d Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Sun, 5 Mar 2023 10:00:47 +0800 Subject: [PATCH 043/404] [fix][broker] Filter system topic when getting topic list by binary proto. (#19667) --- .../pulsar/broker/service/ServerCnx.java | 4 +- .../impl/PatternTopicsConsumerImplTest.java | 54 +++++++++++++++++++ .../pulsar/common/topics/TopicList.java | 4 +- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 5b9c7d39ff12f..071e8a46d5600 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2182,8 +2182,8 @@ protected void handleGetTopicsOfNamespace(CommandGetTopicsOfNamespace commandGet getBrokerService().pulsar().getNamespaceService().getListOfTopics(namespaceName, mode) .thenAccept(topics -> { boolean filterTopics = false; - // filter transaction internal topic - List filteredTopics = TopicList.filterTransactionInternalName(topics); + // filter system topic + List filteredTopics = TopicList.filterSystemTopic(topics); if (enableSubscriptionPatternEvaluation && topicsPattern.isPresent()) { if (topicsPattern.get().length() <= maxSubscriptionPatternLength) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java index 6d19ee1dfe5a1..09a9a003f3ec5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java @@ -63,6 +63,8 @@ public class PatternTopicsConsumerImplTest extends ProducerConsumerBase { public void setup() throws Exception { // set isTcpLookup = true, to use BinaryProtoLookupService to get topics for a pattern. isTcpLookup = true; + // enabled transaction, to test pattern consumers not subscribe to transaction system topic. + conf.setTransactionCoordinatorEnabled(true); super.internalSetup(); super.producerBaseSetup(); } @@ -251,6 +253,58 @@ public void testBinaryProtoToGetTopicsOfNamespacePersistent() throws Exception { producer4.close(); } + @Test(timeOut = testTimeout) + public void testBinaryProtoSubscribeAllTopicOfNamespace() throws Exception { + String key = "testBinaryProtoSubscribeAllTopicOfNamespace"; + String subscriptionName = "my-ex-subscription-" + key; + String topicName1 = "persistent://my-property/my-ns/pattern-topic-1-" + key; + String topicName2 = "persistent://my-property/my-ns/pattern-topic-2-" + key; + String topicName3 = "persistent://my-property/my-ns/pattern-topic-3-" + key; + Pattern pattern = Pattern.compile("my-property/my-ns/.*"); + + // 1. create partition + TenantInfoImpl tenantInfo = createDefaultTenantInfo(); + admin.tenants().createTenant("prop", tenantInfo); + admin.topics().createPartitionedTopic(topicName1, 1); + admin.topics().createPartitionedTopic(topicName2, 2); + admin.topics().createPartitionedTopic(topicName3, 3); + + // 2. create producer to trigger create system topic. + Producer producer = pulsarClient.newProducer().topic(topicName1) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + Consumer consumer = pulsarClient.newConsumer() + .topicsPattern(pattern) + .patternAutoDiscoveryPeriod(2) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) + .receiverQueueSize(4) + .subscribe(); + assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); + + // 4. verify consumer get methods, to get right number of partitions and topics. + assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); + List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); + + assertEquals(topics.size(), 6); + assertEquals(consumers.size(), 6); + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 3); + + topics.forEach(topic -> log.info("topic: {}", topic)); + consumers.forEach(c -> log.info("consumer: {}", c.getTopic())); + + IntStream.range(0, topics.size()).forEach(index -> + assertEquals(consumers.get(index).getTopic(), topics.get(index))); + + consumer.unsubscribe(); + producer.close(); + consumer.close(); + } + @Test(timeOut = testTimeout) public void testPubRateOnNonPersistent() throws Exception { cleanup(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java index 7a01e09ed6461..250cea217ee5f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java @@ -57,9 +57,9 @@ public static List filterTopics(List original, Pattern topicsPat .collect(Collectors.toList()); } - public static List filterTransactionInternalName(List original) { + public static List filterSystemTopic(List original) { return original.stream() - .filter(topic -> !SystemTopicNames.isTransactionInternalName(TopicName.get(topic))) + .filter(topic -> !SystemTopicNames.isSystemTopic(TopicName.get(topic))) .collect(Collectors.toList()); } From e13865cec7f2bc5fb8cbae49b92cfef2e93c515a Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 5 Mar 2023 10:08:30 +0800 Subject: [PATCH 044/404] [fix] [client] fix memory leak if enabled pooled messages (#19585) --- .../impl/BrokerClientIntegrationTest.java | 7 +++- .../pulsar/client/impl/ConsumerBase.java | 3 +- .../pulsar/client/impl/ConsumerImpl.java | 12 +++++++ .../GrowableArrayBlockingQueue.java | 33 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java index c29978c5f5283..716dd1019f4d8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java @@ -947,9 +947,14 @@ public void testPooledMessageWithAckTimeout(boolean isBatchingEnabled) throws Ex ByteBuf payload = ((MessageImpl) msg).getPayload(); assertNotEquals(payload.refCnt(), 0); consumer.redeliverUnacknowledgedMessages(); - assertEquals(payload.refCnt(), 0); + Awaitility.await().untilAsserted(() -> { + assertTrue(consumer.incomingMessages.size() >= 100); + }); consumer.close(); producer.close(); + admin.topics().delete(topic, false); + assertEquals(consumer.incomingMessages.size(), 0); + assertEquals(payload.refCnt(), 0); } /** diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 6988adcccf183..75d3b2edf6e28 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; @@ -82,7 +81,7 @@ public abstract class ConsumerBase extends HandlerState implements Consumer> incomingMessages; + final GrowableArrayBlockingQueue> incomingMessages; protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap; protected final ConcurrentLinkedQueue>> pendingReceives; protected final int maxReceiverQueueSize; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 08a6bb15807c9..18abb5a52c45f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1075,6 +1075,18 @@ private void closeConsumerTasks() { } negativeAcksTracker.close(); stats.getStatTimeout().ifPresent(Timeout::cancel); + if (poolMessages) { + releasePooledMessagesAndStopAcceptNew(); + } + } + + /** + * If enabled pooled messages, we should release the messages after closing consumer and stop accept the new + * messages. + */ + private void releasePooledMessagesAndStopAcceptNew() { + incomingMessages.terminate(message -> message.release()); + clearIncomingMessages(); } void activeConsumerChanged(boolean isActive) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java index 7825f9a0562be..467a455ed8b3d 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java @@ -32,6 +32,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.StampedLock; import java.util.function.Consumer; +import javax.annotation.Nullable; /** * This implements a {@link BlockingQueue} backed by an array with no fixed capacity. @@ -53,6 +54,10 @@ public class GrowableArrayBlockingQueue extends AbstractQueue implements B .newUpdater(GrowableArrayBlockingQueue.class, "size"); private volatile int size = 0; + private volatile boolean terminated = false; + + private volatile Consumer itemAfterTerminatedHandler; + public GrowableArrayBlockingQueue() { this(64); } @@ -132,6 +137,13 @@ public void put(T e) { boolean wasEmpty = false; try { + if (terminated){ + if (itemAfterTerminatedHandler != null) { + itemAfterTerminatedHandler.accept(e); + } + return; + } + if (SIZE_UPDATER.get(this) == data.length) { expandArray(); } @@ -401,6 +413,27 @@ public String toString() { return sb.toString(); } + /** + * Make the queue not accept new items. if there are still new data trying to enter the queue, it will be handed + * by {@param itemAfterTerminatedHandler}. + */ + public void terminate(@Nullable Consumer itemAfterTerminatedHandler) { + // After wait for the in-flight item enqueue, it means the operation of terminate is finished. + long stamp = tailLock.writeLock(); + try { + terminated = true; + if (itemAfterTerminatedHandler != null) { + this.itemAfterTerminatedHandler = itemAfterTerminatedHandler; + } + } finally { + tailLock.unlockWrite(stamp); + } + } + + public boolean isTerminated() { + return terminated; + } + @SuppressWarnings("unchecked") private void expandArray() { // We already hold the tailLock From 4ed8a871bfa9a7a8e30d86d782e33a556646d7b1 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Mon, 6 Mar 2023 12:48:41 +0800 Subject: [PATCH 045/404] [fix][test] testModularLoadManagerRemoveBundleAndLoad (#19710) --- .../pulsar/broker/namespace/NamespaceServiceTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index f4c4799e576f9..ac5d92c880227 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -45,7 +45,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; @@ -823,10 +823,10 @@ private void waitResourceDataUpdateToZK(LoadManager loadManager) throws Exceptio // Wait until "ModularLoadManager" completes processing the ZK notification. ModularLoadManagerWrapper modularLoadManagerWrapper = (ModularLoadManagerWrapper) loadManager; ModularLoadManagerImpl modularLoadManager = (ModularLoadManagerImpl) modularLoadManagerWrapper.getLoadManager(); - ScheduledExecutorService scheduler = (ScheduledExecutorService) FieldUtils.readField( - modularLoadManager, "scheduler", true); + ExecutorService executors = (ExecutorService) FieldUtils.readField( + modularLoadManager, "executors", true); CompletableFuture waitForNoticeHandleFinishByLoadManager = new CompletableFuture<>(); - scheduler.execute(() -> waitForNoticeHandleFinishByLoadManager.complete(null)); + executors.execute(() -> waitForNoticeHandleFinishByLoadManager.complete(null)); waitForNoticeHandleFinishByLoadManager.join(); // Manually trigger "LoadResourceQuotaUpdaterTask" loadManager.writeResourceQuotasToZooKeeper(); From b2658af847511fef29455a22e96baa1cdc9d278f Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 7 Mar 2023 16:21:57 +0800 Subject: [PATCH 046/404] [improve][broker] PIP-192: Make unload and transfer admin API functional (#19538) PIP-192: https://github.com/apache/pulsar/issues/16691 ### Motivation Currently, the unload and transfer admin API still uses the old logic when enabling the `ExtensibleLoadManager`, we need to publish the message to `ServiceUnitStateChannel`, and wait for its response. ### Modifications This PR added a `UnloadManager ` to handle the duplicate unload request and return a `CompletableFuture` to the caller, so we can know when the unload operation is finished. The unload and transfer admin API also been fixed, now we can support do unload when using `ExtensibleLoadManager`. --- .../broker/admin/impl/NamespacesBase.java | 17 +- .../extensions/ExtensibleLoadManagerImpl.java | 53 +++++- .../channel/ServiceUnitStateChannel.java | 8 + .../channel/ServiceUnitStateChannelImpl.java | 24 ++- .../channel/StateChangeListeners.java | 67 +++++++ .../manager/StateChangeListener.java | 33 ++++ .../extensions/manager/UnloadManager.java | 103 ++++++++++ .../extensions/manager/package-info.java | 19 ++ .../extensions/scheduler/UnloadScheduler.java | 17 +- .../broker/namespace/NamespaceService.java | 8 + .../pulsar/broker/web/PulsarWebResource.java | 13 +- .../ExtensibleLoadManagerImplTest.java | 72 +++++++ .../extensions/manager/UnloadManagerTest.java | 178 ++++++++++++++++++ .../scheduler/UnloadSchedulerTest.java | 16 +- 14 files changed, 609 insertions(+), 19 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 30f01ece7de11..5be675f7b636c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -58,6 +58,7 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.loadbalance.LeaderBroker; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; import org.apache.pulsar.broker.service.Subscription; @@ -983,8 +984,17 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang } return CompletableFuture.completedFuture(null); }) - .thenCompose(__ -> validateLeaderBrokerAsync()) + .thenCompose(__ -> { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return CompletableFuture.completedFuture(null); + } + return validateLeaderBrokerAsync(); + }) .thenAccept(__ -> { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return; + } + // For ExtensibleLoadManager, this operation will be ignored. pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, destinationBroker); }); } @@ -1036,10 +1046,11 @@ public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleR namespaceName, bundleRange); return CompletableFuture.completedFuture(null); } + Optional destinationBrokerOpt = Optional.ofNullable(destinationBroker); return validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, authoritative, true) - .thenCompose(nsBundle -> - pulsar().getNamespaceService().unloadNamespaceBundle(nsBundle)); + .thenCompose(nsBundle -> pulsar().getNamespaceService() + .unloadNamespaceBundle(nsBundle, destinationBrokerOpt)); })); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 650c12af57391..2bebe203d8750 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -35,6 +35,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; @@ -43,9 +44,11 @@ import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; +import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; @@ -110,6 +113,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private ScheduledFuture brokerLoadDataReportTask; private ScheduledFuture topBundlesLoadDataReportTask; + private UnloadManager unloadManager; + private boolean started = false; private final AssignCounter assignCounter = new AssignCounter(); @@ -143,6 +148,13 @@ public static boolean isLoadManagerExtensionEnabled(ServiceConfiguration conf) { return ExtensibleLoadManagerImpl.class.getName().equals(conf.getLoadManagerClassName()); } + public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { + if (!(loadManager instanceof ExtensibleLoadManagerWrapper loadManagerWrapper)) { + throw new IllegalArgumentException("The load manager should be 'ExtensibleLoadManagerWrapper'."); + } + return loadManagerWrapper.get(); + } + @Override public void start() throws PulsarServerException { if (this.started) { @@ -151,6 +163,8 @@ public void start() throws PulsarServerException { this.brokerRegistry = new BrokerRegistryImpl(pulsar); this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); this.brokerRegistry.start(); + this.unloadManager = new UnloadManager(); + this.serviceUnitStateChannel.listen(unloadManager); this.serviceUnitStateChannel.start(); try { @@ -201,7 +215,8 @@ public void start() throws PulsarServerException { interval, TimeUnit.MILLISECONDS); // TODO: Start bundle split scheduler. - this.unloadScheduler = new UnloadScheduler(pulsar.getLoadManagerExecutor(), context, serviceUnitStateChannel); + this.unloadScheduler = new UnloadScheduler( + pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel); this.unloadScheduler.start(); this.started = true; } @@ -300,6 +315,12 @@ private CompletableFuture> selectAsync(ServiceUnitId bundle) { @Override public CompletableFuture checkOwnershipAsync(Optional topic, ServiceUnitId bundleUnit) { + return getOwnershipAsync(topic, bundleUnit) + .thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null))); + } + + private CompletableFuture> getOwnershipAsync(Optional topic, + ServiceUnitId bundleUnit) { final String bundle = bundleUnit.toString(); CompletableFuture> owner; if (topic.isPresent() && isInternalTopic(topic.get().toString())) { @@ -307,8 +328,35 @@ public CompletableFuture checkOwnershipAsync(Optional to } else { owner = serviceUnitStateChannel.getOwnerAsync(bundle); } + return owner; + } + + public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, + Optional destinationBroker) { + return getOwnershipAsync(Optional.empty(), bundle) + .thenCompose(brokerOpt -> { + if (brokerOpt.isEmpty()) { + String msg = String.format("Namespace bundle: %s is not owned by any broker.", bundle); + log.warn(msg); + throw new IllegalStateException(msg); + } + String sourceBroker = brokerOpt.get(); + if (destinationBroker.isPresent() && sourceBroker.endsWith(destinationBroker.get())) { + String msg = String.format("Namespace bundle: %s own by %s, cannot be transfer to same broker.", + bundle, sourceBroker); + log.warn(msg); + throw new IllegalArgumentException(msg); + } + return unloadAsync(new Unload(sourceBroker, bundle.toString(), destinationBroker), + conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + }); + } - return owner.thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null))); + private CompletableFuture unloadAsync(Unload unload, + long timeout, + TimeUnit timeoutUnit) { + CompletableFuture future = serviceUnitStateChannel.publishUnloadEventAsync(unload); + return unloadManager.waitAsync(future, unload.serviceUnit(), timeout, timeoutUnit); } @Override @@ -337,6 +385,7 @@ public void close() throws PulsarServerException { try { this.serviceUnitStateChannel.close(); } finally { + this.unloadManager.close(); this.started = false; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java index 44950a21ffd20..dc4d582ddb081 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java @@ -23,6 +23,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.common.stats.Metrics; @@ -156,4 +157,11 @@ public interface ServiceUnitStateChannel extends Closeable { */ List getMetrics(); + /** + * Add a state change listener. + * + * @param listener State change listener. + */ + void listen(StateChangeListener listener); + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index cddaf92d22771..5f24e41dda931 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -72,6 +72,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; @@ -117,6 +118,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private final ConcurrentOpenHashMap> getOwnerRequests; private final String lookupServiceAddress; private final ConcurrentOpenHashMap> cleanupJobs; + private final StateChangeListeners stateChangeListeners; private final LeaderElectionService leaderElectionService; private BrokerSelectionStrategy brokerSelector; private BrokerRegistry brokerRegistry; @@ -191,6 +193,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.getOwnerRequests = ConcurrentOpenHashMap.>newBuilder().build(); this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); + this.stateChangeListeners = new StateChangeListeners(); this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateCleanUpDelayTimeInSeconds() * 1000; this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; @@ -357,6 +360,10 @@ public synchronized void close() throws PulsarServerException { log.info("Successfully cancelled the cleanup tasks"); } + if (stateChangeListeners != null) { + stateChangeListeners.close(); + } + log.info("Successfully closed the channel."); } catch (Exception e) { @@ -619,6 +626,7 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(data.dstBroker()); } + stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); } @@ -628,7 +636,7 @@ private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.dstBroker())) { ServiceUnitStateData next = new ServiceUnitStateData( Owned, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); - pubAsync(serviceUnit, next) + stateChangeListeners.notifyOnCompletion(pubAsync(serviceUnit, next), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } @@ -644,15 +652,15 @@ private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { next = new ServiceUnitStateData( Free, null, data.sourceBroker(), getNextVersionId(data)); } - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) + stateChangeListeners.notifyOnCompletion(closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.sourceBroker())) { - splitServiceUnit(serviceUnit, data) + stateChangeListeners.notifyOnCompletion(splitServiceUnit(serviceUnit, data), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, null)); } } @@ -662,6 +670,7 @@ private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } + stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } @@ -672,6 +681,7 @@ private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); } + stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } @@ -682,6 +692,7 @@ private void handleInitEvent(String serviceUnit) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } + stateChangeListeners.notify(serviceUnit, null, null); log(null, serviceUnit, null, null); } @@ -1302,4 +1313,9 @@ public List getMetrics() { return metrics; } + + @Override + public void listen(StateChangeListener listener) { + this.stateChangeListeners.addListener(listener); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java new file mode 100644 index 0000000000000..1d396f500b648 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; + +@Slf4j +public class StateChangeListeners { + + private final List stateChangeListeners; + + public StateChangeListeners() { + stateChangeListeners = new CopyOnWriteArrayList<>(); + } + + public void addListener(StateChangeListener listener) { + Objects.requireNonNull(listener); + stateChangeListeners.add(listener); + } + + public void close() { + this.stateChangeListeners.clear(); + } + + /** + * Notify all currently added listeners on completion of the future. + * + * @return future of a new completion stage + */ + public CompletableFuture notifyOnCompletion(CompletableFuture future, + String serviceUnit, + ServiceUnitStateData data) { + return future.whenComplete((r, ex) -> notify(serviceUnit, data, ex)); + } + + public void notify(String serviceUnit, ServiceUnitStateData data, Throwable t) { + stateChangeListeners.forEach(listener -> { + try { + listener.handleEvent(serviceUnit, data, t); + } catch (Throwable ex) { + log.error("StateChangeListener: {} exception while handling {} for service unit {}", + listener, data, serviceUnit, ex); + } + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java new file mode 100644 index 0000000000000..7ba8be8771b91 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; + +public interface StateChangeListener { + + /** + * Handle the service unit state change. + * + * @param serviceUnit - Service Unit(Namespace bundle). + * @param data - Service unit state data. + * @param t - Exception, if present. + */ + void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java new file mode 100644 index 0000000000000..ead6384daba8d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; + +/** + * Unload manager. + */ +@Slf4j +public class UnloadManager implements StateChangeListener { + + private final Map> inFlightUnloadRequest; + + public UnloadManager() { + this.inFlightUnloadRequest = new ConcurrentHashMap<>(); + } + + private void complete(String serviceUnit, Throwable ex) { + inFlightUnloadRequest.computeIfPresent(serviceUnit, (__, future) -> { + if (!future.isDone()) { + if (ex != null) { + future.completeExceptionally(ex); + if (log.isDebugEnabled()) { + log.debug("Complete exceptionally unload bundle: {}", serviceUnit, ex); + } + } else { + future.complete(null); + if (log.isDebugEnabled()) { + log.debug("Complete unload bundle: {}", serviceUnit); + } + } + } + return null; + }); + } + + public CompletableFuture waitAsync(CompletableFuture eventPubFuture, + String bundle, + long timeout, + TimeUnit timeoutUnit) { + + return eventPubFuture.thenCompose(__ -> inFlightUnloadRequest.computeIfAbsent(bundle, ignore -> { + if (log.isDebugEnabled()) { + log.debug("Handle unload bundle: {}, timeout: {} {}", bundle, timeout, timeoutUnit); + } + CompletableFuture future = new CompletableFuture<>(); + future.orTimeout(timeout, timeoutUnit).whenComplete((v, ex) -> { + if (ex != null) { + inFlightUnloadRequest.remove(bundle); + log.warn("Failed to wait unload for serviceUnit: {}", bundle, ex); + } + }); + return future; + })); + } + + @Override + public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) { + ServiceUnitState state = ServiceUnitStateData.state(data); + switch (state) { + case Free, Owned -> this.complete(serviceUnit, t); + default -> { + if (log.isDebugEnabled()) { + log.debug("Handling {} for service unit {}", data, serviceUnit); + } + } + } + } + + public void close() { + inFlightUnloadRequest.forEach((bundle, future) -> { + if (!future.isDone()) { + String msg = String.format("Unloading bundle: %s, but the unload manager already closed.", bundle); + log.warn(msg); + future.completeExceptionally(new IllegalStateException(msg)); + } + }); + inFlightUnloadRequest.clear(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java new file mode 100644 index 0000000000000..ac553c0690068 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java index 5cdbd3027104d..e646026978754 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -31,6 +31,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Reflections; @@ -42,6 +43,8 @@ public class UnloadScheduler implements LoadManagerScheduler { private final ScheduledExecutorService loadManagerExecutor; + private final UnloadManager unloadManager; + private final LoadManagerContext context; private final ServiceUnitStateChannel channel; @@ -57,13 +60,16 @@ public class UnloadScheduler implements LoadManagerScheduler { private volatile CompletableFuture currentRunningFuture = null; public UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + UnloadManager unloadManager, LoadManagerContext context, ServiceUnitStateChannel channel) { - this(loadManagerExecutor, context, channel, createNamespaceUnloadStrategy(context.brokerConfiguration())); + this(loadManagerExecutor, unloadManager, context, + channel, createNamespaceUnloadStrategy(context.brokerConfiguration())); } @VisibleForTesting protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + UnloadManager unloadManager, LoadManagerContext context, ServiceUnitStateChannel channel, NamespaceUnloadStrategy strategy) { @@ -71,6 +77,7 @@ protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor, this.recentlyUnloadedBundles = new HashMap<>(); this.recentlyUnloadedBrokers = new HashMap<>(); this.loadManagerExecutor = loadManagerExecutor; + this.unloadManager = unloadManager; this.context = context; this.conf = context.brokerConfiguration(); this.channel = channel; @@ -131,9 +138,11 @@ public synchronized void execute() { List> futures = new ArrayList<>(); unloadDecision.getUnloads().forEach((broker, unload) -> { log.info("[{}] Unloading bundle: {}", namespaceUnloadStrategy.getClass().getSimpleName(), unload); - futures.add(channel.publishUnloadEventAsync(unload).thenAccept(__ -> { - recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); - recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); + futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload), unload.serviceUnit(), + conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS) + .thenAccept(__ -> { + recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); + recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); })); }); return FutureUtil.waitForAll(futures).exceptionally(ex -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 33b15926c3c33..899539e1db6a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -717,6 +717,14 @@ private Optional> getLeastLoadedFromLoadManager(ServiceUnit } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { + return unloadNamespaceBundle(bundle, Optional.empty()); + } + + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, Optional destinationBroker) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + return ExtensibleLoadManagerImpl.get(loadManager.get()) + .unloadNamespaceBundleAsync(bundle, destinationBroker); + } // unload namespace bundle return unloadNamespaceBundle(bundle, config.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 5484a70e1aad0..321a127ad97bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -55,6 +55,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.BookieResources; @@ -626,14 +627,18 @@ protected NamespaceBundle validateNamespaceBundleOwnership(NamespaceName fqnn, B } } - protected CompletableFuture validateNamespaceBundleOwnershipAsync(NamespaceName fqnn, - BundlesData bundles, String bundleRange, boolean authoritative, boolean readOnly) { + protected CompletableFuture validateNamespaceBundleOwnershipAsync( + NamespaceName fqnn, BundlesData bundles, String bundleRange, + boolean authoritative, boolean readOnly) { NamespaceBundle nsBundle; try { nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); } catch (WebApplicationException wae) { return CompletableFuture.failedFuture(wae); } + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return CompletableFuture.completedFuture(nsBundle); + } return validateBundleOwnershipAsync(nsBundle, authoritative, readOnly) .thenApply(__ -> nsBundle); } @@ -992,6 +997,10 @@ protected boolean isLeaderBroker() { } protected static boolean isLeaderBroker(PulsarService pulsar) { + // For extensible load manager, it doesn't have leader election service on pulsar broker. + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar.getConfig())) { + return true; + } return pulsar.getLeaderElectionService().isLeader(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d1bf29725d004..ec82f5c383e2e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -43,10 +43,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.util.LinkedHashMap; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -77,18 +79,22 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TenantResources; import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; @@ -124,10 +130,15 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { public void setup() throws Exception { conf.setAllowAutoTopicCreation(true); conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadBalancerSheddingEnabled(false); super.internalSetup(conf); pulsar1 = pulsar; ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + defaultConf.setLoadBalancerSheddingEnabled(false); additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -148,6 +159,14 @@ public void setup() throws Exception { channel2 = (ServiceUnitStateChannelImpl) FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true); + admin.clusters().createCluster(this.conf.getClusterName(), + ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), + Sets.newHashSet(this.conf.getClusterName()))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); } protected void beforePulsarStart(PulsarService pulsar) throws Exception { @@ -307,6 +326,59 @@ public Map filter(Map broker assertTrue(brokerLookupData.isPresent()); } + @Test(timeOut = 30 * 1000) + public void testUnloadAdminAPI() throws Exception { + TopicName topicName = TopicName.get("test-unload"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + + String broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + + checkOwnershipState(broker, bundle); + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); + assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + + broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + + String dstBrokerUrl = pulsar1.getLookupServiceAddress(); + String dstBrokerServiceUrl; + if (broker.equals(pulsar1.getBrokerServiceUrl())) { + dstBrokerUrl = pulsar2.getLookupServiceAddress(); + dstBrokerServiceUrl = pulsar2.getBrokerServiceUrl(); + } else { + dstBrokerServiceUrl = pulsar1.getBrokerServiceUrl(); + } + checkOwnershipState(broker, bundle); + + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + + assertEquals(admin.lookups().lookupTopic(topicName.toString()), dstBrokerServiceUrl); + + // Test transfer to current broker. + try { + admin.namespaces() + .unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("cannot be transfer to same broker")); + } + } + + private void checkOwnershipState(String broker, NamespaceBundle bundle) + throws ExecutionException, InterruptedException { + var targetLoadManager = secondaryLoadManager; + var otherLoadManager = primaryLoadManager; + if (broker.equals(pulsar1.getBrokerServiceUrl())) { + targetLoadManager = primaryLoadManager; + otherLoadManager = secondaryLoadManager; + } + assertTrue(targetLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + assertFalse(otherLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + } + + @Test public void testGetMetrics() throws Exception { { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java new file mode 100644 index 0000000000000..75ef913b8a851 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.common.util.FutureUtil; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class UnloadManagerTest { + + @Test + public void testEventPubFutureHasException() { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(FutureUtil.failedFuture(new Exception("test")), + "bundle-1", 10, TimeUnit.SECONDS); + + assertTrue(future.isCompletedExceptionally()); + try { + future.get(); + fail(); + } catch (Exception ex) { + assertEquals(ex.getCause().getMessage(), "test"); + } + } + + @Test + public void testTimeout() throws IllegalAccessException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 3, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + try { + future.get(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getCause() instanceof TimeoutException); + } + + assertEquals(inFlightUnloadRequestMap.size(), 0); + } + + @Test + public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Assigning, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Deleted, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Init, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Free, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 0); + future.get(); + + // Success with Owned state. + future = manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 0); + future.get(); + } + + @Test + public void testFailedStage() throws IllegalAccessException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), + new IllegalStateException("Failed stage.")); + + try { + future.get(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getCause() instanceof IllegalStateException); + assertEquals(ex.getCause().getMessage(), "Failed stage."); + } + + assertEquals(inFlightUnloadRequestMap.size(), 0); + } + + @Test + public void testClose() throws IllegalAccessException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + assertEquals(inFlightUnloadRequestMap.size(), 1); + manager.close(); + assertEquals(inFlightUnloadRequestMap.size(), 0); + + try { + future.get(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getCause() instanceof IllegalStateException); + } + } + + private Map> getInFlightUnloadRequestMap(UnloadManager manager) + throws IllegalAccessException { + Map> inFlightUnloadRequest = + (Map>) FieldUtils.readField(manager, "inFlightUnloadRequest", true); + + return inFlightUnloadRequest; + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java index cda5f81d81b93..73d4eb1f18bfb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.scheduler; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -31,6 +32,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.client.util.ExecutorProvider; @@ -71,17 +73,20 @@ public void testExecuteSuccess() { LoadManagerContext context = setupContext(); BrokerRegistry registry = context.brokerRegistry(); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + UnloadManager unloadManager = mock(UnloadManager.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); doReturn(CompletableFuture.completedFuture(Lists.newArrayList("broker-1", "broker-2"))) .when(registry).getAvailableBrokersAsync(); doReturn(CompletableFuture.completedFuture(null)).when(channel).publishUnloadEventAsync(any()); + doReturn(CompletableFuture.completedFuture(null)).when(unloadManager) + .waitAsync(any(), any(), anyLong(), any()); UnloadDecision decision = new UnloadDecision(); Unload unload = new Unload("broker-1", "bundle-1"); decision.getUnloads().put("broker-1", unload); doReturn(decision).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); scheduler.execute(); @@ -101,6 +106,7 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio LoadManagerContext context = setupContext(); BrokerRegistry registry = context.brokerRegistry(); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + UnloadManager unloadManager = mock(UnloadManager.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); doAnswer(__ -> CompletableFuture.supplyAsync(() -> { @@ -112,7 +118,7 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio } return Lists.newArrayList("broker-1", "broker-2"); }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync(); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); ExecutorService executorService = Executors.newFixedThreadPool(10); CountDownLatch latch = new CountDownLatch(10); @@ -133,7 +139,8 @@ public void testDisableLoadBalancer() { context.brokerConfiguration().setLoadBalancerEnabled(false); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadManager unloadManager = mock(UnloadManager.class); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); scheduler.execute(); @@ -152,7 +159,8 @@ public void testNotChannelOwner() { context.brokerConfiguration().setLoadBalancerEnabled(false); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadManager unloadManager = mock(UnloadManager.class); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); doReturn(CompletableFuture.completedFuture(false)).when(channel).isChannelOwnerAsync(); scheduler.execute(); From af1360fb167c1f9484fda5771df3ea9b21d1440b Mon Sep 17 00:00:00 2001 From: sinan liu <54846009+Denovo1998@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:34:32 +0800 Subject: [PATCH 047/404] [PIP-236][fix][broker]Fix using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic (#17449) Fixes #17354 PIP #19113 ### Motivation *Fixed the failure to use schema to create consumer after using AUTO-CONSUME consumer to subscribe an empty topic, and Broker returned the error message as `IncompatibleSchemaException("Topic does not have schema to check")`.* ### Modifications *In PersistentTopic::addSchemaIfIdleOrCheckCompatible, when there is an active consumer, but the consumer is using the AUTO_CONSUME schema to subscribe to the topic. Continuing to create a schema consumer to subscribe to the topic will fail.* - When `numActiveConsumers != 0`, and check the schema of the currently existing consumers is AUTO_CONSUME schema. --- .../pulsar/broker/service/Consumer.java | 16 ++++ .../pulsar/broker/service/ServerCnx.java | 3 +- .../broker/service/SubscriptionOption.java | 2 + .../nonpersistent/NonPersistentTopic.java | 21 ++++-- .../service/persistent/PersistentTopic.java | 42 ++++++----- .../pulsar/client/api/SimpleSchemaTest.java | 73 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 6 ++ .../client/impl/schema/AutoConsumeSchema.java | 6 ++ .../pulsar/common/protocol/Commands.java | 12 ++- pulsar-common/src/main/proto/PulsarApi.proto | 2 + 10 files changed, 153 insertions(+), 30 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 47da95b34ac3f..1ee3f513ef288 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -58,6 +58,7 @@ import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -142,12 +143,24 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; + @Getter + private final SchemaType schemaType; + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, boolean isDurable, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch) { + this(subscription, subType, topicName, consumerId, priorityLevel, consumerName, isDurable, cnx, appId, + metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, null); + } + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, + int priorityLevel, String consumerName, + boolean isDurable, TransportCnx cnx, String appId, + Map metadata, boolean readCompacted, + KeySharedMeta keySharedMeta, MessageId startMessageId, + long consumerEpoch, SchemaType schemaType) { this.subscription = subscription; this.subType = subType; this.topicName = topicName; @@ -204,6 +217,8 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.consumerEpoch = consumerEpoch; this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); + + this.schemaType = schemaType; } @VisibleForTesting @@ -231,6 +246,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; + this.schemaType = null; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 071e8a46d5600..4fc79a124acd8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1195,8 +1195,9 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) + .schemaType(schema == null ? null : schema.getType()) .build(); - if (schema != null) { + if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { return topic.addSchemaIfIdleOrCheckCompatible(schema) .thenCompose(v -> topic.subscribe(option)); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java index d375c539e550e..af56d023616b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java @@ -29,6 +29,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; @Getter @Builder @@ -49,6 +50,7 @@ public class SubscriptionOption { private KeySharedMeta keySharedMeta; private Optional> subscriptionProperties; private long consumerEpoch; + private SchemaType schemaType; public static Optional> getPropertiesMap(List list) { if (list == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 3b046570d732e..a0a8462a22753 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -255,7 +256,8 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), - option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null)); + option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), + option.getSchemaType()); } @Override @@ -268,7 +270,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null); + replicateSubscriptionState, keySharedMeta, null, null); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -279,7 +281,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St long resetStartMessageBackInSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - Map subscriptionProperties) { + Map subscriptionProperties, + SchemaType schemaType) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -321,8 +324,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St name -> new NonPersistentSubscription(this, subscriptionName, isDurable, subscriptionProperties)); Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, - false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } @@ -1162,12 +1165,14 @@ public CompletableFuture getLastMessageId() { @Override public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { return hasSchema().thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) + int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().stream() + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .toList().size()) .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumersWithoutAutoSchema != 0) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4277edb074c5e..fd0b069421275 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -155,6 +155,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicCompactionStrategy; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; @@ -727,7 +728,8 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), - option.getSubscriptionProperties().orElse(Collections.emptyMap()), option.getConsumerEpoch()); + option.getSubscriptionProperties().orElse(Collections.emptyMap()), + option.getConsumerEpoch(), option.getSchemaType()); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -740,7 +742,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicatedSubscriptionStateArg, KeySharedMeta keySharedMeta, Map subscriptionProperties, - long consumerEpoch) { + long consumerEpoch, + SchemaType schemaType) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -828,7 +831,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaType); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { if (subscription instanceof PersistentSubscription persistentSubscription) { @@ -907,7 +910,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, null); } private CompletableFuture getDurableSubscription(String subscriptionName, @@ -3107,21 +3110,22 @@ public synchronized OffloadProcessStatus offloadStatus() { @Override public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { - return hasSchema() - .thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) - .sum(); - if (hasSchema - || (!producers.isEmpty()) - || (numActiveConsumers != 0) - || (ledger.getTotalSize() != 0)) { - return checkSchemaCompatibleForConsumer(schema); - } else { - return addSchema(schema).thenCompose(schemaVersion -> - CompletableFuture.completedFuture(null)); - } - }); + return hasSchema().thenCompose((hasSchema) -> { + int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().stream() + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .toList().size()) + .sum(); + if (hasSchema + || (!producers.isEmpty()) + || (numActiveConsumersWithoutAutoSchema != 0) + || (ledger.getTotalSize() != 0)) { + return checkSchemaCompatibleForConsumer(schema); + } else { + return addSchema(schema).thenCompose(schemaVersion -> + CompletableFuture.completedFuture(null)); + } + }); } public synchronized void checkReplicatedSubscriptionControllerState() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index 21d6a7ed89a56..c8c7c3b2ccc38 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1239,6 +1239,79 @@ public void testAutoCreatedSchema(String domain) throws Exception { Assert.assertEquals(admin.schemas().getSchemaInfo(topic2).getType(), SchemaType.STRING); } + @Test(dataProvider = "topicDomain") + public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throws Exception { + final String topic = domain + "my-property/my-ns/testSubscribeWithSchemaAfterAutoConsume-1"; + + @Cleanup + Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer1") + .subscribe(); + @Cleanup + Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer2") + .subscribe(); + @Cleanup + Consumer autoConsumer3 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("autoConsumer3") + .subscribe(); + @Cleanup + Consumer autoConsumer4 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("autoConsumer4") + .subscribe(); + try { + log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); + log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + log.info("The autoConsumer3 isConnected: " + autoConsumer3.isConnected()); + log.info("The autoConsumer4 isConnected: " + autoConsumer4.isConnected()); + admin.schemas().getSchemaInfo(topic); + fail("The schema of topic should not exist"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + @Cleanup + Consumer consumerWithSchema1 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema-1") + .subscribe(); + @Cleanup + Consumer consumerWithSchema2 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema-2") + .subscribe(); + @Cleanup + Consumer consumerWithSchema3 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("consumerWithSchema-3") + .subscribe(); + @Cleanup + Consumer consumerWithSchema4 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("consumerWithSchema-4") + .subscribe(); + } + @DataProvider(name = "keyEncodingType") public static Object[] keyEncodingType() { return new Object[] { KeyValueEncodingType.SEPARATED, KeyValueEncodingType.INLINE }; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 18abb5a52c45f..beaa34bf20520 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -84,6 +84,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.RetryMessageUtil; @@ -811,6 +812,11 @@ public void connectionOpened(final ClientCnx cnx) { if (si != null && (SchemaType.BYTES == si.getType() || SchemaType.NONE == si.getType())) { // don't set schema for Schema.BYTES si = null; + } else { + if (schema instanceof AutoConsumeSchema + && Commands.peerSupportsCarryAutoConsumeSchemaToBroker(cnx.getRemoteEndpointProtocolVersion())) { + si = AutoConsumeSchema.SCHEMA_INFO; + } } // startMessageRollbackDurationInSec should be consider only once when consumer connects to first time long startMessageRollbackDuration = (startMessageRollbackDurationInSec > 0 diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java index 33fcd18876be6..82a3b69da20b6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java @@ -57,6 +57,12 @@ public class AutoConsumeSchema implements Schema { private SchemaInfoProvider schemaInfoProvider; + public static final SchemaInfo SCHEMA_INFO = SchemaInfoImpl.builder() + .name("AutoConsume") + .type(SchemaType.AUTO_CONSUME) + .schema(new byte[0]) + .build(); + private ConcurrentMap> initSchemaMap() { ConcurrentMap> schemaMap = new ConcurrentHashMap<>(); // The Schema.BYTES will not be uploaded to the broker and store in the schema storage, diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 081dfe4275b24..8a5684cf676b0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -771,7 +771,9 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, } private static Schema.Type getSchemaType(SchemaType type) { - if (type.getValue() < 0) { + if (type == SchemaType.AUTO_CONSUME) { + return Schema.Type.AutoConsume; + } else if (type.getValue() < 0) { return Schema.Type.None; } else { return Schema.Type.valueOf(type.getValue()); @@ -779,7 +781,9 @@ private static Schema.Type getSchemaType(SchemaType type) { } public static SchemaType getSchemaType(Schema.Type type) { - if (type.getValue() < 0) { + if (type == Schema.Type.AutoConsume) { + return SchemaType.AUTO_CONSUME; + } else if (type.getValue() < 0) { // this is unexpected return SchemaType.NONE; } else { @@ -1965,6 +1969,10 @@ public static boolean peerSupportsAckReceipt(int peerVersion) { return peerVersion >= ProtocolVersion.v17.getValue(); } + public static boolean peerSupportsCarryAutoConsumeSchemaToBroker(int peerVersion) { + return peerVersion >= ProtocolVersion.v21.getValue(); + } + private static org.apache.pulsar.common.api.proto.ProducerAccessMode convertProducerAccessMode( ProducerAccessMode accessMode) { switch (accessMode) { diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index acf75eab85826..d9c41eeec9740 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,6 +45,7 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; + AutoConsume = 21; } required string name = 1; @@ -263,6 +264,7 @@ enum ProtocolVersion { v18 = 18; // Add client support for broker entry metadata v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse v20 = 20; // Add client support for topic migration redirection CommandTopicMigrated + v21 = 21; // Carry the AUTO_CONSUME schema to the Broker after this version } message CommandConnect { From e973388eb77f226e0546f88c87821f8c7ee26281 Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Wed, 8 Mar 2023 16:48:11 +0800 Subject: [PATCH 048/404] [fix][broker] Fixed history load not releasing (#19726) --- .../loadbalance/ModularLoadManagerStrategy.java | 7 +++++++ .../impl/LeastResourceUsageWithWeight.java | 9 +++++++-- .../impl/ModularLoadManagerImpl.java | 1 + .../ModularLoadManagerStrategyTest.java | 17 +++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java index 2be8200aef5c1..91a619eafc226 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java @@ -46,6 +46,13 @@ public interface ModularLoadManagerStrategy { Optional selectBroker(Set candidates, BundleData bundleToAssign, LoadData loadData, ServiceConfiguration conf); + /** + * Triggered when active brokers change. + */ + default void onActiveBrokersChange(Set activeBrokers) { + + } + /** * Create a placement strategy using the configuration. * diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java index f50f9ed36538a..ab3e63e9d133f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java @@ -128,8 +128,9 @@ private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData * @return The name of the selected broker as it appears on ZooKeeper. */ @Override - public Optional selectBroker(Set candidates, BundleData bundleToAssign, LoadData loadData, - ServiceConfiguration conf) { + public synchronized Optional selectBroker(Set candidates, BundleData bundleToAssign, + LoadData loadData, + ServiceConfiguration conf) { if (candidates.isEmpty()) { log.info("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign); return Optional.empty(); @@ -167,4 +168,8 @@ public Optional selectBroker(Set candidates, BundleData bundleTo } return Optional.of(bestBrokers.get(ThreadLocalRandom.current().nextInt(bestBrokers.size()))); } + @Override + public synchronized void onActiveBrokersChange(Set activeBrokers) { + brokerAvgResourceUsageWithWeight.keySet().removeIf((key) -> !activeBrokers.contains(key)); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index ea2472eb199d4..59f9836b30975 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -494,6 +494,7 @@ private void cleanupDeadBrokersData() { for (LoadSheddingStrategy loadSheddingStrategy : loadSheddingPipeline) { loadSheddingStrategy.onActiveBrokersChange(activeBrokers); } + placementStrategy.onActiveBrokersChange(activeBrokers); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java index b967deaa72686..c64c9950a95a9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java @@ -228,6 +228,23 @@ public void testRoundRobinBrokerSelector() throws IllegalAccessException { assertEquals(((AtomicInteger) FieldUtils.readDeclaredField(strategy, "count", true)).get(), 0); } + public void testActiveBrokersChange() throws Exception { + LoadData loadData = new LoadData(); + Map brokerDataMap = loadData.getBrokerData(); + brokerDataMap.put("1", initBrokerData()); + brokerDataMap.put("2", initBrokerData()); + brokerDataMap.put("3", initBrokerData()); + ServiceConfiguration conf = new ServiceConfiguration(); + LeastResourceUsageWithWeight strategy = new LeastResourceUsageWithWeight(); + strategy.selectBroker(brokerDataMap.keySet(), new BundleData(), loadData, conf); + Field field = LeastResourceUsageWithWeight.class.getDeclaredField("brokerAvgResourceUsageWithWeight"); + field.setAccessible(true); + Map map = (Map) field.get(strategy); + assertEquals(map.size(), 3); + strategy.onActiveBrokersChange(new HashSet<>()); + assertEquals(map.size(), 0); + } + private BrokerData initBrokerData(double usage, double limit) { LocalBrokerData localBrokerData = new LocalBrokerData(); localBrokerData.setCpu(new ResourceUsage(usage, limit)); From cbd799f05eb26aeac5ffd5bbb9751ad9e5928dd3 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 9 Mar 2023 10:17:39 +0800 Subject: [PATCH 049/404] [fix][meta] Fix deadlock causes session notification not to work (#19754) ### Motivation This is a namespace bundle double-owners problem. We found it in the memory dumps. The memory dumps show that the notification thread has been blocked for a long time by the leader election deadlock, And many notifications are blocked in the executor queue. This causes we can't to revalidate the locks, and they are still thinking them working well. For private reasons, I can't share the namespace bundle snapshot, but the blocked thread easily explains it. image ^^ blocked thread image ^^ executor queue ### Modifications - Avoid putting the new task to single thread executor causes deadlock. --- .../coordination/impl/LeaderElectionImpl.java | 10 ++- .../impl/LeaderElectionImplTest.java | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index 409d49dcd26ab..ad2a5bef70610 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -19,6 +19,7 @@ package org.apache.pulsar.metadata.coordination.impl; import com.fasterxml.jackson.databind.type.TypeFactory; +import com.google.common.annotations.VisibleForTesting; import java.util.EnumSet; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -111,13 +112,13 @@ private synchronized CompletableFuture elect() { } else { return tryToBecomeLeader(); } - }).thenComposeAsync(leaderElectionState -> { + }).thenCompose(leaderElectionState -> { // make sure that the cache contains the current leader // so that getLeaderValueIfPresent works on all brokers cache.refresh(path); return cache.get(path) .thenApply(__ -> leaderElectionState); - }, executor); + }); } private synchronized CompletableFuture handleExistingLeaderValue(GetResult res) { @@ -336,4 +337,9 @@ private void handlePathNotification(Notification notification) { } } } + + @VisibleForTesting + protected ScheduledExecutorService getSchedulerExecutor() { + return executor; + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java new file mode 100644 index 0000000000000..027521d2ffc17 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.coordination.impl; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; +import lombok.Cleanup; +import org.apache.pulsar.metadata.BaseMetadataStoreTest; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.coordination.CoordinationService; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.testng.annotations.Test; + +public class LeaderElectionImplTest extends BaseMetadataStoreTest { + + @Test(dataProvider = "impl", timeOut = 10000) + public void validateDeadLock(String provider, Supplier urlSupplier) + throws Exception { + if (provider.equals("Memory") || provider.equals("RocksDB")) { + // There are no multiple sessions for the local memory provider + return; + } + + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElectionImpl le = (LeaderElectionImpl) cs.getLeaderElection(String.class, + path, __ -> { + }); + final CompletableFuture blockFuture = new CompletableFuture<>(); + // simulate handleSessionNotification method logic + le.getSchedulerExecutor().execute(() -> { + try { + le.elect("test-2").join(); + blockFuture.complete(null); + } catch (Throwable ex) { + blockFuture.completeExceptionally(ex); + } + }); + blockFuture.join(); + } +} From cdeef00c5f6a5bd3197b4ca6de0a0505b18835d8 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Mar 2023 10:38:21 +0800 Subject: [PATCH 050/404] [fix] [broker] Topic close failure leaves subscription in a permanent fence state (#19692) Motivation : After a Topic close failure or a delete failure, the fence state will be reset to get the topic back to work,but it will not reset the fence state of the subscription, which will result in the consumer never being able to connect to the broker. Modifications: Reset the fence state of subscriptions if the operation of topic close is failed. --- .../persistent/PersistentSubscription.java | 39 ++++++++++--- .../service/persistent/PersistentTopic.java | 1 + .../persistent/PersistentTopicTest.java | 55 +++++++++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index dfeca26750342..e07d7bee500a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -126,6 +126,7 @@ public class PersistentSubscription extends AbstractSubscription implements Subs private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; private final PendingAckHandle pendingAckHandle; private volatile Map subscriptionProperties; + private volatile CompletableFuture fenceFuture; static { REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); @@ -897,7 +898,10 @@ public CompletableFuture close() { */ @Override public synchronized CompletableFuture disconnect() { - CompletableFuture disconnectFuture = new CompletableFuture<>(); + if (fenceFuture != null){ + return fenceFuture; + } + fenceFuture = new CompletableFuture<>(); // block any further consumers on this subscription IS_FENCED_UPDATER.set(this, TRUE); @@ -905,19 +909,38 @@ public synchronized CompletableFuture disconnect() { (dispatcher != null ? dispatcher.close() : CompletableFuture.completedFuture(null)) .thenCompose(v -> close()).thenRun(() -> { log.info("[{}][{}] Successfully disconnected and closed subscription", topicName, subName); - disconnectFuture.complete(null); + fenceFuture.complete(null); }).exceptionally(exception -> { - IS_FENCED_UPDATER.set(this, FALSE); - if (dispatcher != null) { - dispatcher.reset(); - } log.error("[{}][{}] Error disconnecting consumers from subscription", topicName, subName, exception); - disconnectFuture.completeExceptionally(exception); + fenceFuture.completeExceptionally(exception); + resumeAfterFence(); return null; }); + return fenceFuture; + } - return disconnectFuture; + /** + * Resume subscription after topic deletion or close failure. + */ + public synchronized void resumeAfterFence() { + // If "fenceFuture" is null, it means that "disconnect" has never been called. + if (fenceFuture != null) { + fenceFuture.whenComplete((ignore, ignoreEx) -> { + synchronized (PersistentSubscription.this) { + try { + if (IS_FENCED_UPDATER.compareAndSet(this, TRUE, FALSE)) { + if (dispatcher != null) { + dispatcher.reset(); + } + } + fenceFuture = null; + } catch (Exception ex) { + log.error("[{}] Resume subscription [{}] failure", topicName, subName, ex); + } + } + }); + } } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index fd0b069421275..3b9fbbceb2422 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3234,6 +3234,7 @@ private void fenceTopicToCloseOrDelete() { } private void unfenceTopicToResume() { + subscriptions.values().forEach(sub -> sub.resumeAfterFence()); isFenced = false; isClosingOrDeleting = false; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 565157508683f..80a79e0234de4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -19,7 +19,10 @@ package org.apache.pulsar.broker.service.persistent; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -39,11 +42,15 @@ import java.util.Collection; import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.Cleanup; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.stats.PrometheusMetricsTest; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; @@ -470,4 +477,52 @@ public void testCompatibilityWithPartitionKeyword() throws PulsarAdminException, Assert.assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.topics().getPartitionedTopicMetadata(topicName)); } + + @Test + public void testDeleteTopicFail() throws Exception { + final String fullyTopicName = "persistent://prop/ns-abc/" + "tp_" + + UUID.randomUUID().toString().replaceAll("-", ""); + // Mock topic. + BrokerService brokerService = spy(pulsar.getBrokerService()); + doReturn(brokerService).when(pulsar).getBrokerService(); + + // Create a sub, and send one message. + Consumer consumer1 = pulsarClient.newConsumer(Schema.STRING).topic(fullyTopicName).subscriptionName("sub1") + .subscribe(); + consumer1.close(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(fullyTopicName).create(); + producer.send("1"); + producer.close(); + + // Make a failed delete operation. + AtomicBoolean makeDeletedFailed = new AtomicBoolean(true); + PersistentTopic persistentTopic = (PersistentTopic) brokerService.getTopic(fullyTopicName, false).get().get(); + doAnswer(invocation -> { + CompletableFuture future = (CompletableFuture) invocation.getArguments()[1]; + if (makeDeletedFailed.get()) { + future.completeExceptionally(new RuntimeException("mock ex for test")); + } else { + future.complete(null); + } + return null; + }).when(brokerService) + .deleteTopicAuthenticationWithRetry(any(String.class), any(CompletableFuture.class), anyInt()); + try { + persistentTopic.delete().get(); + } catch (Exception e) { + org.testng.Assert.assertTrue(e instanceof ExecutionException); + org.testng.Assert.assertTrue(e.getCause() instanceof java.lang.RuntimeException); + org.testng.Assert.assertEquals(e.getCause().getMessage(), "mock ex for test"); + } + + // Assert topic works after deleting failure. + Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING).topic(fullyTopicName).subscriptionName("sub1") + .subscribe(); + org.testng.Assert.assertEquals("1", consumer2.receive(2, TimeUnit.SECONDS).getValue()); + consumer2.close(); + + // Make delete success. + makeDeletedFailed.set(false); + persistentTopic.delete().get(); + } } From 401fb055ed428b7f3e33a49cea78b25b04f7448e Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Mar 2023 15:13:50 +0800 Subject: [PATCH 051/404] [fix] [broker] delete topic failed if disabled system topic (#19735) Motivation: After PR #18823, The cmd delete topic will fail if disabled the feature system topic. Modifications: do not delete the system policy if disabled the feature system topic --- .../service/persistent/PersistentTopic.java | 3 +- .../client/impl/DisabledSystemTopicTest.java | 61 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 3b9fbbceb2422..79717ed57303f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1246,7 +1246,8 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, deleteTopicAuthenticationFuture.thenCompose(ignore -> deleteSchema()) .thenCompose(ignore -> { - if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { + if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic) + && brokerService.getPulsar().getConfiguration().isSystemTopicEnabled()) { return deleteTopicPolicies(); } else { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java new file mode 100644 index 0000000000000..7747d3a576c90 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class DisabledSystemTopicTest extends ProducerConsumerBase { + + @Override + @BeforeMethod + public void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterMethod(alwaysRun = true) + public void cleanup() throws Exception { + super.internalCleanup(); + } + + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setTransactionCoordinatorEnabled(false); + conf.setSystemTopicEnabled(false); + } + + @Test + public void testDeleteTopic() throws Exception { + String topicName = "persistent://my-property/my-ns/tp_" + UUID.randomUUID().toString(); + + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().delete(topicName, false); + + admin.topics().createPartitionedTopic(topicName, 3); + admin.topics().deletePartitionedTopic(topicName); + } +} From 4a450aa1091d15db14e3d69454c3bb89b3ae6d7d Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 9 Mar 2023 16:18:00 +0800 Subject: [PATCH 052/404] [fix][broker] Fix potential exception cause the policy service init fail. (#19746) --- .../SystemTopicBasedTopicPoliciesService.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 9ec374264e91c..9b10055f36fac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -341,20 +341,19 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp return; } if (hasMore) { - reader.readNextAsync().whenComplete((msg, e) -> { - if (e != null) { - log.error("[{}] Failed to read event from the system topic.", - reader.getSystemTopic().getTopicName(), e); - future.completeExceptionally(e); - cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); - return; - } + reader.readNextAsync().thenAccept(msg -> { refreshTopicPoliciesCache(msg); if (log.isDebugEnabled()) { log.debug("[{}] Loop next event reading for system topic.", reader.getSystemTopic().getTopicName().getNamespaceObject()); } initPolicesCache(reader, future); + }).exceptionally(e -> { + log.error("[{}] Failed to read event from the system topic.", + reader.getSystemTopic().getTopicName(), e); + future.completeExceptionally(e); + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); + return null; }); } else { if (log.isDebugEnabled()) { From d4930a31c052dd8fcd5982b649898967a24f8961 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 9 Mar 2023 01:09:34 -0800 Subject: [PATCH 053/404] [fix][io] KCA: 'desanitize' topic name for the pulsar's ctx calls (#19756) --- .../io/kafka/connect/KafkaConnectSink.java | 22 +++++++++- .../connect/PulsarKafkaSinkTaskContext.java | 14 ++++-- .../kafka/connect/KafkaConnectSinkTest.java | 44 +++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index 37d0987e61023..efbad2ef47ae0 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -95,6 +95,12 @@ public class KafkaConnectSink implements Sink { CacheBuilder.newBuilder().maximumSize(1000) .expireAfterAccess(30, TimeUnit.MINUTES).build(); + // Can't really safely expire these entries. If we do, we could end up with + // a sanitized topic name that used in e.g. resume() after a long pause but can't be + // // re-resolved into a form usable for Pulsar. + private final Cache desanitizedTopicCache = + CacheBuilder.newBuilder().build(); + private int maxBatchBitsForOffset = 12; private boolean useIndexAsOffset = true; @@ -184,7 +190,18 @@ public void open(Map config, SinkContext ctx) throws Exception { }); task = (SinkTask) taskClass.getConstructor().newInstance(); taskContext = - new PulsarKafkaSinkTaskContext(configs.get(0), ctx, task::open); + new PulsarKafkaSinkTaskContext(configs.get(0), ctx, task::open, kafkaName -> { + if (sanitizeTopicName) { + String pulsarTopicName = desanitizedTopicCache.getIfPresent(kafkaName); + if (log.isDebugEnabled()) { + log.debug("desanitizedTopicCache got: kafkaName: {}, pulsarTopicName: {}", + kafkaName, pulsarTopicName); + } + return pulsarTopicName != null ? pulsarTopicName : kafkaName; + } else { + return kafkaName; + } + }); task.initialize(taskContext); task.start(configs.get(0)); @@ -486,6 +503,9 @@ protected String sanitizeNameIfNeeded(String name, boolean sanitize) { if (sanitizedName.matches("^[^a-zA-Z_].*")) { sanitizedName = "_" + sanitizedName; } + // do this once, sanitize() can be called on already sanitized name + // so avoid replacing with (sanitizedName -> sanitizedName). + desanitizedTopicCache.get(sanitizedName, () -> name); return sanitizedName; }); } catch (ExecutionException e) { diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java index 99a8bf2908237..7a908b553a89a 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java @@ -33,6 +33,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; @@ -49,6 +50,7 @@ public class PulsarKafkaSinkTaskContext implements SinkTaskContext { private final SinkContext ctx; private final OffsetBackingStore offsetStore; + private Function desanitizeTopicName; private final String topicNamespace; private final Consumer> onPartitionChange; private final AtomicBoolean runRepartition = new AtomicBoolean(false); @@ -57,11 +59,13 @@ public class PulsarKafkaSinkTaskContext implements SinkTaskContext { public PulsarKafkaSinkTaskContext(Map config, SinkContext ctx, - Consumer> onPartitionChange) { + Consumer> onPartitionChange, + Function desanitizeTopicName) { this.config = config; this.ctx = ctx; offsetStore = new PulsarOffsetBackingStore(ctx.getPulsarClient()); + this.desanitizeTopicName = desanitizeTopicName; PulsarKafkaWorkerConfig pulsarKafkaWorkerConfig = new PulsarKafkaWorkerConfig(config); offsetStore.configure(pulsarKafkaWorkerConfig); offsetStore.start(); @@ -144,7 +148,9 @@ private void fillOffsetMap(Map offsetMap, TopicPartition private void seekAndUpdateOffset(TopicPartition topicPartition, long offset) { try { - ctx.seek(topicPartition.topic(), topicPartition.partition(), MessageIdUtils.getMessageId(offset)); + ctx.seek(desanitizeTopicName.apply(topicPartition.topic()), + topicPartition.partition(), + MessageIdUtils.getMessageId(offset)); } catch (PulsarClientException e) { log.error("Failed to seek topic {} partition {} offset {}", topicPartition.topic(), topicPartition.partition(), offset, e); @@ -202,7 +208,7 @@ public Set assignment() { public void pause(TopicPartition... topicPartitions) { for (TopicPartition tp: topicPartitions) { try { - ctx.pause(tp.topic(), tp.partition()); + ctx.pause(desanitizeTopicName.apply(tp.topic()), tp.partition()); } catch (PulsarClientException e) { log.error("Failed to pause topic {} partition {}", tp.topic(), tp.partition(), e); throw new RuntimeException("Failed to pause topic " + tp.topic() + " partition " + tp.partition(), e); @@ -214,7 +220,7 @@ public void pause(TopicPartition... topicPartitions) { public void resume(TopicPartition... topicPartitions) { for (TopicPartition tp: topicPartitions) { try { - ctx.resume(tp.topic(), tp.partition()); + ctx.resume(desanitizeTopicName.apply(tp.topic()), tp.partition()); } catch (PulsarClientException e) { log.error("Failed to resume topic {} partition {}", tp.topic(), tp.partition(), e); throw new RuntimeException("Failed to resume topic " + tp.topic() + " partition " + tp.partition(), e); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index fe2def250233c..567562d338b98 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -314,6 +314,50 @@ public void seekPauseResumeTest() throws Exception { sink.close(); } + @Test + public void seekPauseResumeWithSanitizeTest() throws Exception { + KafkaConnectSink sink = new KafkaConnectSink(); + props.put("sanitizeTopicName", "true"); + sink.open(props, context); + + String pulsarTopicName = "persistent://a-b/c-d/fake-topic.a"; + + final GenericRecord rec = getGenericRecord("value", Schema.STRING); + Message msg = mock(MessageImpl.class); + when(msg.getValue()).thenReturn(rec); + final MessageId msgId = new MessageIdImpl(10, 10, 0); + when(msg.getMessageId()).thenReturn(msgId); + + final AtomicInteger status = new AtomicInteger(0); + Record record = PulsarRecord.builder() + .topicName(pulsarTopicName) + .message(msg) + .ackFunction(status::incrementAndGet) + .failFunction(status::decrementAndGet) + .schema(Schema.STRING) + .build(); + + sink.write(record); + sink.flush(); + + assertEquals(status.get(), 1); + + final TopicPartition tp = new TopicPartition(sink.sanitizeNameIfNeeded(pulsarTopicName, true), 0); + assertNotEquals(MessageIdUtils.getOffset(msgId), 0); + assertEquals(sink.currentOffset(tp.topic(), tp.partition()), MessageIdUtils.getOffset(msgId)); + + sink.taskContext.offset(tp, 0); + verify(context, times(1)).seek(pulsarTopicName, + tp.partition(), MessageIdUtils.getMessageId(0)); + assertEquals(sink.currentOffset(tp.topic(), tp.partition()), 0); + + sink.taskContext.pause(tp); + verify(context, times(1)).pause(pulsarTopicName, tp.partition()); + sink.taskContext.resume(tp); + verify(context, times(1)).resume(pulsarTopicName, tp.partition()); + + sink.close(); + } @Test public void subscriptionTypeTest() throws Exception { From 7d5d2df0f0caf7630e89e4aab79a87e632475ab8 Mon Sep 17 00:00:00 2001 From: HuangZeGui Date: Thu, 9 Mar 2023 19:35:58 +0800 Subject: [PATCH 054/404] [fix][doc] Update the comment of the method of OwnershipCache (#19553) Co-authored-by: huangzegui --- .../org/apache/pulsar/broker/namespace/OwnershipCache.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index a9dd44d4589d9..7d0b5a4147721 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -158,8 +158,7 @@ public CompletableFuture checkOwnershipAsync(NamespaceBundle bundle) { * @param suName * name of the ServiceUnit * @return The ephemeral node data showing the current ownership info in ZooKeeper - * @throws Exception - * throws exception if no ownership info is found + * or empty if no ownership info is found */ public CompletableFuture> getOwnerAsync(NamespaceBundle suName) { CompletableFuture ownedBundleFuture = ownedBundlesCache.getIfPresent(suName); From 42a65aaa55e03c147adabac55ad5227fc8c8f596 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:27:06 +0800 Subject: [PATCH 055/404] [fix][test]testRedirectOfGetCoordinatorInternalStats (#19715) Fix https://github.com/apache/pulsar/issues/18600 ### Motivation Reason for test failure: 1. Create 16 brokers and 3 transaction coordinator topics. 2. The 3 transaction coordinator topics are assigned to broker-0. 3. Because the admin is connected to broker-0, we do not hope the topics are assigned to broker-0. 4. Recreate the 3 transaction coordinator topics. 5. The topics are assigned to broker-0 again. 6. Repeat execute steps 3,4,5 7. Timeout ### Modifications Do not recreate the transaction coordinator topics. Recreate an admin connect the broker which is not the owner of the transaction coordinator topics. --- .../AdminApiTransactionMultiBrokerTest.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java index 463a65fc8ae89..52aadde7b2621 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar.broker.admin.v3; +import static org.mockito.Mockito.spy; import java.util.Map; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.naming.SystemTopicNames; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -46,21 +47,26 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + /** + * This test is used to verify the redirect request of `getCoordinatorInternalStats`. + *

+ * 1. Set up 16 broker and create 3 transaction coordinator topic. + * 2. The 3 transaction coordinator topic will be assigned to these brokers through some kind of + * load-balancing strategy. (In current implementations, they tend to be assigned to a broker.) + * 3. Find a broker x which is not the owner of the transaction coordinator topic. + * 4. Create a admin connected to broker x, and use the admin to call ` getCoordinatorInternalStats`. + *

+ */ @Test public void testRedirectOfGetCoordinatorInternalStats() throws Exception { - Map map = admin.lookups() + PulsarAdmin localAdmin = this.admin; + Map map = localAdmin.lookups() .lookupPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.toString()); - while (map.containsValue(getPulsarServiceList().get(0).getBrokerServiceUrl())) { - pulsarServiceList.get(0).getPulsarResources() - .getNamespaceResources() - .getPartitionedTopicResources() - .deletePartitionedTopicAsync(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN); - pulsarServiceList.get(0).getPulsarResources() - .getNamespaceResources() - .getPartitionedTopicResources() - .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, - new PartitionedTopicMetadata(NUM_PARTITIONS)); - map = admin.lookups().lookupPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.toString()); + + for (int i = 0; map.containsValue(getPulsarServiceList().get(i).getBrokerServiceUrl()); i++) { + if (!map.containsValue(getPulsarServiceList().get(i + 1).getBrokerServiceUrl())) + localAdmin = spy(createNewPulsarAdmin(PulsarAdmin.builder() + .serviceHttpUrl(pulsarServiceList.get(i + 1).getWebServiceAddress()))); } if (pulsarClient != null) { pulsarClient.shutdown(); @@ -72,7 +78,7 @@ public void testRedirectOfGetCoordinatorInternalStats() throws Exception { .enableTransaction(true) .build(); for (int i = 0; i < NUM_PARTITIONS; i++) { - admin.transactions().getCoordinatorInternalStats(i, false); + localAdmin.transactions().getCoordinatorInternalStats(i, false); } } } From bae28f90c39bd85a25a35e6195440c7558251ddb Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Mar 2023 23:07:48 +0800 Subject: [PATCH 056/404] [fix] [doc] fix multiple apis in the automatically generated documentation use the same anchor point (#19193) Generate operationId using the specified rule `{className}_{methodName}` of swagger. --- pulsar-broker/pom.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 8fc1527a6e3e8..7442d95e467e6 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -697,11 +697,13 @@ com.github.kongchen swagger-maven-plugin - 3.1.7 + 3.1.8 false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v2.Bookies org.apache.pulsar.broker.admin.v2.BrokerStats @@ -737,6 +739,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.lookup.v2 http,https /lookup @@ -754,6 +758,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Functions http,https /admin/v3 @@ -771,6 +777,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Transactions http,https /admin/v3 @@ -788,6 +796,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Sources http,https /admin/v3 @@ -805,6 +815,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Sinks http,https /admin/v3 @@ -822,6 +834,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Packages http,https /admin/v3 From da30b2e211d0a097abfa8d0392f1fd4b13a46328 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 9 Mar 2023 07:36:09 -0800 Subject: [PATCH 057/404] [StructuredEventLog] Added support for Log4j2 structured logging if available (#16353) --- structured-event-log/pom.xml | 7 + .../structuredeventlog/Initializer.java | 44 ++++ .../StructuredEventLog.java | 6 +- .../log4j2/Log4j2Event.java | 208 ++++++++++++++++++ .../log4j2/Log4j2StructuredEventLog.java | 46 ++++ .../log4j2/package-info.java | 19 ++ .../slf4j/StructuredEventLogTest.java | 32 +-- 7 files changed, 342 insertions(+), 20 deletions(-) create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 6daaa21991515..8c058306fc278 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -37,6 +37,13 @@ org.slf4j slf4j-api + + + + org.apache.logging.log4j + log4j-core + provided + org.hamcrest hamcrest diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java new file mode 100644 index 0000000000000..bb78c21e75631 --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog; + +import org.apache.pulsar.structuredeventlog.log4j2.Log4j2StructuredEventLog; +import org.apache.pulsar.structuredeventlog.slf4j.Slf4jStructuredEventLog; + +class Initializer { + static StructuredEventLog get() { + return INSTANCE; + } + + private static final StructuredEventLog INSTANCE; + + static { + StructuredEventLog log = null; + try { + // Use Log4j2 if available in the classpath + Class.forName("org.apache.logging.log4j.LogManager"); + log = Log4j2StructuredEventLog.INSTANCE; + } catch (Throwable t) { + // Fallback to Slf4j otherwise + log = Slf4jStructuredEventLog.INSTANCE; + } + + INSTANCE = log; + } +} diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java index bc64545416b7f..30e6e417f0448 100644 --- a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.structuredeventlog; -import org.apache.pulsar.structuredeventlog.slf4j.Slf4jStructuredEventLog; - /** * Structured event logging interface * @@ -85,7 +83,7 @@ public interface StructuredEventLog { /** * Create a new logger object, from which root events can be created. */ - static StructuredEventLog newLogger() { - return Slf4jStructuredEventLog.INSTANCE; + static StructuredEventLog get() { + return Initializer.get(); } } diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java new file mode 100644 index 0000000000000..187d2064b3888 --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog.log4j2; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.StringMapMessage; +import org.apache.pulsar.structuredeventlog.Event; +import org.apache.pulsar.structuredeventlog.EventResources; +import org.apache.pulsar.structuredeventlog.EventResourcesImpl; + +class Log4j2Event implements Event { + private static final Logger stringLogger = LogManager.getLogger(); + + private final Clock clock; + private String traceId = null; + private String parentId = null; + private List attributes = null; + private Level level = Level.INFO; + private Throwable throwable = null; + private Instant startTime = null; + private final EventResourcesImpl resources; + + Log4j2Event(Clock clock, EventResourcesImpl parentResources) { + this.clock = clock; + this.resources = new EventResourcesImpl(parentResources); + } + + @Override + public Event newChildEvent() { + return new Log4j2Event(clock, resources).traceId(traceId); + } + + @Override + public Event traceId(String traceId) { + this.traceId = traceId; + return this; + } + + @Override + public Event parentId(String parentId) { + this.parentId = parentId; + return this; + } + + @Override + public Event timed() { + startTime = clock.instant(); + return this; + } + + @Override + public Event sampled(Object samplingKey, int duration, TimeUnit unit) { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public Event resources(EventResources other) { + if (other instanceof EventResourcesImpl) { + this.resources.copyFrom((EventResourcesImpl) other); + } + return this; + } + + @Override + public Event resource(String key, Object value) { + resources.resource(key, value); + return this; + } + + @Override + public Event resource(String key, Supplier value) { + resources.resource(key, value); + return this; + } + + @Override + public Event attr(String key, Object value) { + getAttributes().add(key); + getAttributes().add(value); + return this; + } + + @Override + public Event attr(String key, Supplier value) { + this.attr(key, (Object) value); + return this; + } + + @Override + public Event exception(Throwable t) { + this.throwable = t; + return this; + } + + @Override + public Event atError() { + this.level = Level.ERROR; + return this; + } + + @Override + public Event atInfo() { + this.level = Level.INFO; + return this; + } + + @Override + public Event atWarn() { + this.level = Level.WARN; + return this; + } + + @Override + public void log(Enum event) { + throw new UnsupportedOperationException(); + } + + @Override + public void log(String event) { + logInternal(stringLogger, event); + } + + private void logInternal(Logger logger, String msg) { + StringMapMessage event = new StringMapMessage(); + event.with("msg", msg); + if (traceId != null) { + event.with("traceId", traceId); + } + if (parentId != null) { + event.with("parentId", parentId); + } + resources.forEach(event::with); + if (attributes != null) { + EventResourcesImpl.forEach(attributes, event::with); + } + if (startTime != null) { + event.with("startTimestamp", startTime.toString()); + event.with("durationMs", String.valueOf(Duration.between(startTime, clock.instant()).toMillis())); + } + switch (level) { + case ERROR: + if (throwable != null) { + + logger.error(event, throwable); + } else { + logger.error(event); + } + break; + case WARN: + if (throwable != null) { + logger.warn(event, throwable); + } else { + logger.warn(event); + } + break; + case INFO: + default: + if (throwable != null) { + logger.info(event, throwable); + } else { + logger.info(event); + } + break; + } + } + + @Override + public void stash() { + throw new UnsupportedOperationException("TODO"); + } + + private List getAttributes() { + if (attributes == null) { + attributes = new ArrayList<>(); + } + return attributes; + } + + enum Level { + INFO, + WARN, + ERROR + } +} diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java new file mode 100644 index 0000000000000..e854a814e14f1 --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog.log4j2; + +import java.time.Clock; +import org.apache.pulsar.structuredeventlog.Event; +import org.apache.pulsar.structuredeventlog.EventResources; +import org.apache.pulsar.structuredeventlog.EventResourcesImpl; +import org.apache.pulsar.structuredeventlog.StructuredEventLog; + +public class Log4j2StructuredEventLog implements StructuredEventLog { + public static final Log4j2StructuredEventLog INSTANCE = new Log4j2StructuredEventLog(); + // Visible for testing + Clock clock = Clock.systemUTC(); + + @Override + public Event newRootEvent() { + return new Log4j2Event(clock, null); + } + + @Override + public EventResources newEventResources() { + return new EventResourcesImpl(null); + } + + @Override + public Event unstash() { + throw new UnsupportedOperationException("TODO"); + } +} diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java new file mode 100644 index 0000000000000..c784880755abb --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog.log4j2; diff --git a/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java b/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java index 2da8294784432..48662b59ba923 100644 --- a/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java +++ b/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java @@ -76,7 +76,7 @@ public void setupLog4j() throws Exception { @Test public void testTraceId() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; Event e = log.newRootEvent(); e.newChildEvent().log("child"); @@ -97,7 +97,7 @@ public void testTraceId() throws Exception { @Test public void testParentId() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; Event e1 = log.newRootEvent(); Event e2 = e1.newChildEvent(); @@ -124,7 +124,7 @@ public void testParentId() throws Exception { @Test public void testResources() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; EventResources res = log.newEventResources() .resource("r1", "v1") @@ -167,7 +167,7 @@ public void testResources() throws Exception { @Test public void testResourcesNullTest() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; EventResources res = log.newEventResources() .resource(null, "v1") @@ -205,7 +205,7 @@ public void testResourcesNullTest() throws Exception { @Test public void testAttributes() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; Event e1 = log.newRootEvent() .attr("a1", "v1") @@ -238,7 +238,7 @@ public void testAttributes() throws Exception { @Test public void testAttributedNullTest() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent() .attr(null, "v1") @@ -262,7 +262,7 @@ public void testAttributedNullTest() throws Exception { @Test public void testInfoLevel() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().log("info1"); log.newRootEvent().atInfo().log("info2"); @@ -281,7 +281,7 @@ public void testInfoLevel() throws Exception { @SuppressWarnings("unchecked") @Test public void testInfoLevelException() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().exception(new Throwable("cause1")).log("info1"); log.newRootEvent().atInfo().exception(new Throwable("cause2")).log("info2"); @@ -296,7 +296,7 @@ public void testInfoLevelException() throws Exception { @Test public void testWarnLevel() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atWarn().log("warn1"); @@ -308,7 +308,7 @@ public void testWarnLevel() throws Exception { @SuppressWarnings("unchecked") @Test public void testWarnLevelException() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atWarn().exception(new Throwable("cause1")).log("warn1"); @@ -319,7 +319,7 @@ public void testWarnLevelException() throws Exception { @Test public void testErrorLevel() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atError().log("error1"); @@ -331,7 +331,7 @@ public void testErrorLevel() throws Exception { @SuppressWarnings("unchecked") @Test public void testErrorLevelException() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atError().exception(new Throwable("cause1")).log("error1"); @@ -344,8 +344,8 @@ public void testErrorLevelException() throws Exception { @Test public void testTimedEvent() throws Exception { MockClock clock = new MockClock(); - StructuredEventLog log = StructuredEventLog.newLogger(); - ((Slf4jStructuredEventLog)log).clock = clock; + Slf4jStructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; + log.clock = clock; Event e = log.newRootEvent().timed(); clock.advanceTime(1234, TimeUnit.MILLISECONDS); e.log("timed"); @@ -363,7 +363,7 @@ public enum Events { @Test public void testEventGroups() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().log(Events.TEST_EVENT); List> logged = getLogged(); @@ -379,7 +379,7 @@ public enum BareEvents { @Test public void testBareEnum() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().log(BareEvents.BARE_EVENT); List> logged = getLogged(); From 9feb85b19ca160b4be3acbfe15f39edde07608f5 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Fri, 10 Mar 2023 16:21:25 +0800 Subject: [PATCH 058/404] [fix][client] Fix topic list watcher fail log (#19733) --- .../java/org/apache/pulsar/client/impl/TopicListWatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java index 88ada6a344b8d..384d1b688b8d5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java @@ -148,8 +148,8 @@ public void connectionOpened(ClientCnx cnx) { cnx.channel().close(); return null; } - log.warn("[{}][{}] Failed to subscribe to topic on {}", topic, - getHandlerName(), cnx.channel().remoteAddress()); + log.warn("[{}][{}] Failed to create topic list watcher on {}", + topic, getHandlerName(), cnx.channel().remoteAddress()); if (e.getCause() instanceof PulsarClientException && PulsarClientException.isRetriableError(e.getCause()) From 90b0f0a17579d22d413853ed4941d81debbe0cbe Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Fri, 10 Mar 2023 01:58:40 -0800 Subject: [PATCH 059/404] [fix][io] KCA: Option to use kafka connector's SourceConnector class to create task and task config (#19772) --- .../connect/AbstractKafkaConnectSource.java | 48 ++++++++++++++--- .../kafka/connect/KafkaConnectSourceTest.java | 54 ++++++++++++------- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java index 6b4ae9d080257..36d2b4bcdbf0d 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.io.kafka.connect; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import io.confluent.connect.avro.AvroConverter; import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; @@ -33,7 +35,9 @@ import java.util.concurrent.atomic.AtomicInteger; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.runtime.TaskConfig; +import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.source.SourceTaskContext; @@ -55,6 +59,7 @@ public abstract class AbstractKafkaConnectSource implements Source { // kafka connect related variables private SourceTaskContext sourceTaskContext; + private SourceConnector connector; @Getter private SourceTask sourceTask; public Converter keyConverter; @@ -71,6 +76,8 @@ public abstract class AbstractKafkaConnectSource implements Source { // number of outstandingRecords that have been polled but not been acked private final AtomicInteger outstandingRecords = new AtomicInteger(0); + public static final String CONNECTOR_CLASS = "kafkaConnectorSourceClass"; + @Override public void open(Map config, SourceContext sourceContext) throws Exception { Map stringConfig = new HashMap<>(); @@ -80,12 +87,6 @@ public void open(Map config, SourceContext sourceContext) throws } }); - // get the source class name from config and create source task from reflection - sourceTask = Class.forName(stringConfig.get(TaskConfig.TASK_CLASS_CONFIG)) - .asSubclass(SourceTask.class) - .getDeclaredConstructor() - .newInstance(); - topicNamespace = stringConfig.get(PulsarKafkaWorkerConfig.TOPIC_NAMESPACE_CONFIG); // initialize the key and value converter @@ -129,8 +130,36 @@ public void open(Map config, SourceContext sourceContext) throws sourceTaskContext = new PulsarIOSourceTaskContext(offsetReader, pulsarKafkaWorkerConfig); + final Map taskConfig; + if (config.get(CONNECTOR_CLASS) != null) { + String kafkaConnectorFQClassName = config.get(CONNECTOR_CLASS).toString(); + Class clazz = Class.forName(kafkaConnectorFQClassName); + connector = (SourceConnector) clazz.getConstructor().newInstance(); + + Class taskClass = connector.taskClass(); + sourceTask = (SourceTask) taskClass.getConstructor().newInstance(); + + connector.initialize(new PulsarKafkaSinkContext()); + connector.start(stringConfig); + + List> configs = connector.taskConfigs(1); + checkNotNull(configs); + checkArgument(configs.size() == 1); + taskConfig = configs.get(0); + } else { + // for backward compatibility with old configuration + // that use the task directly + + // get the source class name from config and create source task from reflection + sourceTask = Class.forName(stringConfig.get(TaskConfig.TASK_CLASS_CONFIG)) + .asSubclass(SourceTask.class) + .getDeclaredConstructor() + .newInstance(); + taskConfig = stringConfig; + } + sourceTask.initialize(sourceTaskContext); - sourceTask.start(stringConfig); + sourceTask.start(taskConfig); } @Override @@ -178,6 +207,11 @@ public void close() { sourceTask = null; } + if (connector != null) { + connector.stop(); + connector = null; + } + if (offsetStore != null) { offsetStore.stop(); offsetStore = null; diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java index 7dcf3ce8393df..8852ba02b040f 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java @@ -23,7 +23,6 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import java.io.File; import java.io.OutputStream; import java.nio.file.Files; @@ -47,7 +46,6 @@ @Slf4j public class KafkaConnectSourceTest extends ProducerConsumerBase { - private Map config = new HashMap<>(); private String offsetTopicName; // The topic to publish data to, for kafkaSource private String topicName; @@ -62,18 +60,10 @@ protected void setup() throws Exception { super.internalSetup(); super.producerBaseSetup(); - config.put(TaskConfig.TASK_CLASS_CONFIG, "org.apache.kafka.connect.file.FileStreamSourceTask"); - config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); - config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); - this.offsetTopicName = "persistent://my-property/my-ns/kafka-connect-source-offset"; - config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); - this.topicName = "persistent://my-property/my-ns/kafka-connect-source"; - config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); tempFile = File.createTempFile("some-file-name", null); - config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); - config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + tempFile.deleteOnExit(); this.context = mock(SourceContext.class); this.client = PulsarClient.builder() @@ -91,16 +81,44 @@ protected void cleanup() throws Exception { tempFile.delete(); super.internalCleanup(); } - protected void completedFlush(Throwable error, Void result) { - if (error != null) { - log.error("Failed to flush {} offsets to storage: ", this, error); - } else { - log.info("Finished flushing {} offsets to storage", this); - } + + @Test + public void testOpenAndReadConnectorConfig() throws Exception { + Map config = getConfig(); + config.put(AbstractKafkaConnectSource.CONNECTOR_CLASS, + "org.apache.kafka.connect.file.FileStreamSourceConnector"); + + testOpenAndReadTask(config); } @Test - public void testOpenAndRead() throws Exception { + public void testOpenAndReadTaskDirect() throws Exception { + Map config = getConfig(); + + config.put(TaskConfig.TASK_CLASS_CONFIG, + "org.apache.kafka.connect.file.FileStreamSourceTask"); + + testOpenAndReadTask(config); + } + + private Map getConfig() { + Map config = new HashMap<>(); + + config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, + "org.apache.kafka.connect.storage.StringConverter"); + config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, + "org.apache.kafka.connect.storage.StringConverter"); + + config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); + + config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); + config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); + config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, + String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + return config; + } + + private void testOpenAndReadTask(Map config) throws Exception { kafkaConnectSource = new KafkaConnectSource(); kafkaConnectSource.open(config, context); From bfc620bf6f2586b099fe423d7b07cc88754ef312 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 11 Mar 2023 13:02:53 +0800 Subject: [PATCH 060/404] [fix] [test] fix flaky test BucketDelayedDeliveryTrackerTest. testWithBkException (#19751) This test sets the maximum number of buckets to 10, then creates multiple buckets by writing, then uses the merge mechanism to make the final number of buckets less than or equal to 10. But `BucketDelayedDeliveryTracker` doesn't guarantee that every merger operation will work, these cases will make the operation fail: - the last persistention of the bucket has not finished. - all buckets are full. So remove this validation --- .../delayed/bucket/BucketDelayedDeliveryTrackerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 08e1f78725bf4..0ba9e5f4ca2e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -308,10 +308,6 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) { assertEquals(110, tracker.getNumberOfDelayedMessages()); - int size = tracker.getImmutableBuckets().asMapOfRanges().size(); - - assertEquals(10, size); - tracker.addMessage(111, 1011, 111 * 10); MutableLong delayedMessagesInSnapshot = new MutableLong(); From b6a73823ce2ed47baa606b2ecc5f1569e4c101a5 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Sat, 11 Mar 2023 22:59:29 +0800 Subject: [PATCH 061/404] [improve][broker] PIP-192: Support broker isolation policy (#19592) --- .../extensions/ExtensibleLoadManagerImpl.java | 10 +- .../extensions/filter/BrokerFilter.java | 12 +- .../filter/BrokerIsolationPoliciesFilter.java | 60 +++++ .../filter/BrokerMaxTopicCountFilter.java | 8 + .../filter/BrokerVersionFilter.java | 11 +- .../policies/IsolationPoliciesHelper.java | 68 ++++++ .../extensions/policies/package-info.java | 19 ++ .../extensions/scheduler/TransferShedder.java | 79 +++++- .../ExtensibleLoadManagerImplTest.java | 61 ++++- .../BrokerIsolationPoliciesFilterTest.java | 222 +++++++++++++++++ .../filter/BrokerMaxTopicCountFilterTest.java | 2 +- .../filter/BrokerVersionFilterTest.java | 13 +- .../scheduler/TransferShedderTest.java | 227 ++++++++++++++++-- 13 files changed, 750 insertions(+), 42 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 2bebe203d8750..82790f44fcb28 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -42,6 +42,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; @@ -139,6 +140,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline = new ArrayList<>(); this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); + this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); // TODO: Make brokerSelectionStrategy configurable. this.brokerSelectionStrategy = new LeastResourceUsageWithWeight(); @@ -225,6 +227,7 @@ public void start() throws PulsarServerException { public void initialize(PulsarService pulsar) { this.pulsar = pulsar; this.conf = pulsar.getConfiguration(); + this.brokerFilterPipeline.forEach(brokerFilter -> brokerFilter.initialize(pulsar)); } @Override @@ -287,7 +290,6 @@ private CompletableFuture> selectAsync(ServiceUnitId bundle) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() .thenCompose(availableBrokers -> { - // TODO: Support isolation policies LoadManagerContext context = this.getContext(); Map availableBrokerCandidates = new HashMap<>(availableBrokers); @@ -296,11 +298,13 @@ private CompletableFuture> selectAsync(ServiceUnitId bundle) { List filterPipeline = getBrokerFilterPipeline(); for (final BrokerFilter filter : filterPipeline) { try { - filter.filter(availableBrokerCandidates, context); + filter.filter(availableBrokerCandidates, bundle, context); + // Preserve the filter successes result. + availableBrokers.keySet().retainAll(availableBrokerCandidates.keySet()); } catch (BrokerFilterException e) { // TODO: We may need to revisit this error case. log.error("Failed to filter out brokers.", e); - availableBrokerCandidates = availableBrokers; + availableBrokerCandidates = new HashMap<>(availableBrokers); } } if (availableBrokerCandidates.isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index 35f4b6817f131..30d25f559b11e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,9 +19,11 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; /** * Filter out unqualified Brokers, which are not entered into LoadBalancer for decision-making. @@ -33,14 +35,22 @@ public interface BrokerFilter { */ String name(); + /** + * Initialize this broker filter using the given pulsar service. + */ + void initialize(PulsarService pulsar); + /** * Filter out unqualified brokers based on implementation. * * @param brokers The full broker and lookup data. + * @param serviceUnit The current serviceUnit. * @param context The load manager context. * @return Filtered broker list. */ - Map filter(Map brokers, LoadManagerContext context) + Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java new file mode 100644 index 0000000000000..b28c77f76f3eb --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.ServiceUnitId; + + +@Slf4j +public class BrokerIsolationPoliciesFilter implements BrokerFilter { + + public static final String FILTER_NAME = "broker_isolation_policies_filter"; + + private IsolationPoliciesHelper isolationPoliciesHelper; + + @Override + public String name() { + return FILTER_NAME; + } + + @Override + public void initialize(PulsarService pulsar) { + this.isolationPoliciesHelper = new IsolationPoliciesHelper(new SimpleResourceAllocationPolicies(pulsar)); + } + + @Override + public Map filter(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) + throws BrokerFilterException { + Set brokerCandidateCache = + isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, serviceUnit); + availableBrokers.keySet().retainAll(brokerCandidateCache); + return availableBrokers; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index e3f8faca32468..b98edd3d425e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,10 +20,12 @@ import java.util.Map; import java.util.Optional; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; public class BrokerMaxTopicCountFilter implements BrokerFilter { @@ -34,8 +36,14 @@ public String name() { return FILTER_NAME; } + @Override + public void initialize(PulsarService pulsar) { + // No-op + } + @Override public Map filter(Map brokers, + ServiceUnitId serviceUnit, LoadManagerContext context) throws BrokerFilterException { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 869fb049a3cd8..b7332a5ff10a0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -22,11 +22,13 @@ import java.util.Iterator; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; /** * Filter by broker version. @@ -45,7 +47,9 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public Map filter(Map brokers, LoadManagerContext context) + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { @@ -144,4 +148,9 @@ public Version getLatestVersionNumber(Map brokerMap) public String name() { return FILTER_NAME; } + + @Override + public void initialize(PulsarService pulsar) { + // No-op + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java new file mode 100644 index 0000000000000..4d7a5bf22d661 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.policies; + +import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.ServiceUnitId; + +@Slf4j +public class IsolationPoliciesHelper { + + private final SimpleResourceAllocationPolicies policies; + + public IsolationPoliciesHelper(SimpleResourceAllocationPolicies policies) { + this.policies = policies; + } + + private static final FastThreadLocal> localBrokerCandidateCache = new FastThreadLocal<>() { + @Override + protected Set initialValue() { + return new HashSet<>(); + } + }; + + public Set applyIsolationPolicies(Map availableBrokers, + ServiceUnitId serviceUnit) { + Set brokerCandidateCache = localBrokerCandidateCache.get(); + brokerCandidateCache.clear(); + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + availableBrokers.keySet(), new LoadManagerShared.BrokerTopicLoadingPredicate() { + @Override + public boolean isEnablePersistentTopics(String brokerUrl) { + BrokerLookupData lookupData = availableBrokers.get(brokerUrl.replace("http://", "")); + return lookupData != null && lookupData.persistentTopicsEnabled(); + } + + @Override + public boolean isEnableNonPersistentTopics(String brokerUrl) { + BrokerLookupData lookupData = availableBrokers.get(brokerUrl.replace("http://", "")); + return lookupData != null && lookupData.nonPersistentTopicsEnabled(); + } + }); + return brokerCandidateCache; + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java new file mode 100644 index 0000000000000..76e0c4a894290 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.policies; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 7f9128de81754..810fda320af68 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -21,10 +21,15 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.MinMaxPriorityQueue; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.Getter; import lombok.experimental.Accessors; import org.apache.commons.lang3.StringUtils; @@ -32,12 +37,15 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; @@ -71,6 +79,7 @@ public class TransferShedder implements NamespaceUnloadStrategy { private final LoadStats stats = new LoadStats(); private final PulsarService pulsar; private final SimpleResourceAllocationPolicies allocationPolicies; + private final IsolationPoliciesHelper isolationPoliciesHelper; private final UnloadDecision decision = new UnloadDecision(); @@ -78,11 +87,13 @@ public class TransferShedder implements NamespaceUnloadStrategy { public TransferShedder(){ this.pulsar = null; this.allocationPolicies = null; + this.isolationPoliciesHelper = null; } public TransferShedder(PulsarService pulsar){ this.pulsar = pulsar; this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); + this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); } @@ -265,6 +276,17 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd(); boolean transfer = conf.isLoadBalancerTransferEnabled(); + + Map availableBrokers; + try { + availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync() + .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + decision.skip(Unknown); + log.warn("Failed to fetch available brokers. Reason:{}. Stop unloading.", decision.getReason(), e); + return decision; + } + while (true) { if (!stats.hasTransferableBrokers()) { if (debugMode) { @@ -334,7 +356,9 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, int remainingTopBundles = topBundlesLoadData.size(); for (var e : topBundlesLoadData) { String bundle = e.bundleName(); - if (!recentlyUnloadedBundles.containsKey(bundle) && isTransferable(bundle)) { + if (!recentlyUnloadedBundles.containsKey(bundle) + && isTransferable(context, availableBrokers, + bundle, maxBroker, Optional.of(minBroker))) { var bundleData = e.stats(); double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut; if (remainingTopBundles > 1 @@ -342,8 +366,7 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, || !atLeastOneBundleSelected)) { if (transfer) { selectedBundlesCache.put(maxBroker, - new Unload(maxBroker, bundle, - Optional.of(minBroker))); + new Unload(maxBroker, bundle, Optional.of(minBroker))); } else { selectedBundlesCache.put(maxBroker, new Unload(maxBroker, bundle)); @@ -412,18 +435,27 @@ private boolean hasMsgThroughput(LoadManagerContext context, String broker) { } - private boolean isTransferable(String bundle) { + private boolean isTransferable(LoadManagerContext context, + Map availableBrokers, + String bundle, + String maxBroker, + Optional broker) { if (pulsar == null || allocationPolicies == null) { return true; } - NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle)); - if (allocationPolicies.areIsolationPoliciesPresent(namespace)) { + String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + NamespaceName namespaceName = NamespaceName.get(namespace); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + NamespaceBundle namespaceBundle = + pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespace, bundleRange); + + if (!canTransferWithIsolationPoliciesToBroker(context, availableBrokers, namespaceBundle, maxBroker, broker)) { return false; } try { var localPoliciesOptional = pulsar - .getPulsarResources().getLocalPolicies().getLocalPolicies(namespace); + .getPulsarResources().getLocalPolicies().getLocalPolicies(namespaceName); if (localPoliciesOptional.isPresent() && StringUtils.isNotBlank( localPoliciesOptional.get().namespaceAntiAffinityGroup)) { return false; @@ -434,4 +466,37 @@ private boolean isTransferable(String bundle) { } return true; } + + /** + * Check the gave bundle and broker can be transfer or unload with isolation policies applied. + * + * @param context The load manager context. + * @param availableBrokers The available brokers. + * @param namespaceBundle The bundle try to unload or transfer. + * @param currentBroker The current broker. + * @param targetBroker The broker will be transfer to. + * @return Can be transfer/unload or not. + */ + private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext context, + Map availableBrokers, + NamespaceBundle namespaceBundle, + String currentBroker, + Optional targetBroker) { + if (isolationPoliciesHelper == null + || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) { + return true; + } + boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled(); + Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle); + + // Remove the current bundle owner broker. + candidates.remove(currentBroker); + + // Unload: Check if there are any more candidates available for selection. + if (targetBroker.isEmpty() || !transfer) { + return !candidates.isEmpty(); + } + // Transfer: Check if this broker is among the candidates. + return candidates.contains(targetBroker.get()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index ec82f5c383e2e..441415a9d35e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -90,6 +90,7 @@ import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; @@ -290,12 +291,19 @@ public String name() { return "Mock broker filter"; } + @Override + public void initialize(PulsarService pulsar) { + // No-op + } + @Override public Map filter(Map brokers, - LoadManagerContext context) { + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { brokers.remove(pulsar1.getLookupServiceAddress()); return brokers; } + })).when(primaryLoadManager).getBrokerFilterPipeline(); Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); @@ -308,14 +316,10 @@ public void testFilterHasException() throws Exception { TopicName topicName = TopicName.get("test-filter-has-exception"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); - doReturn(List.of(new BrokerFilter() { - @Override - public String name() { - return "Mock broker filter"; - } - + doReturn(List.of(new MockBrokerFilter() { @Override public Map filter(Map brokers, + ServiceUnitId serviceUnit, LoadManagerContext context) throws BrokerFilterException { brokers.clear(); throw new BrokerFilterException("Test"); @@ -379,6 +383,35 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle) } + @Test + public void testMoreThenOneFilter() throws Exception { + TopicName topicName = TopicName.get("test-filter-has-exception"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + + String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); + doReturn(List.of(new MockBrokerFilter() { + @Override + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + brokers.remove(lookupServiceAddress1); + return brokers; + } + },new MockBrokerFilter() { + @Override + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + brokers.clear(); + throw new BrokerFilterException("Test"); + } + })).when(primaryLoadManager).getBrokerFilterPipeline(); + + Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); + assertTrue(brokerLookupData.isPresent()); + assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress()); + } + @Test public void testGetMetrics() throws Exception { { @@ -565,6 +598,20 @@ SplitDecision.Reason.Balanced, new MutableLong(6) assertEquals(actual, expected); } + private static abstract class MockBrokerFilter implements BrokerFilter { + + @Override + public String name() { + return "Mock-broker-filter"; + } + + @Override + public void initialize(PulsarService pulsar) { + // No-op + } + + } + private static void cleanTableView(ServiceUnitStateChannel channel) throws IllegalAccessException { var tv = (TableViewImpl) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java new file mode 100644 index 0000000000000..a079a23bcea04 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang.reflect.FieldUtils; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; +import org.testng.annotations.Test; + +/** + * Unit test for {@link BrokerIsolationPoliciesFilter}. + */ +@Test(groups = "broker") +public class BrokerIsolationPoliciesFilterTest { + + /** + * It verifies namespace-isolation policies with primary and secondary brokers. + * + * usecase: + * + * 
+     *  1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1
+     *     a. available-brokers: broker1, broker2, broker3 => result: broker1
+     *     b. available-brokers: broker2, broker3          => result: broker2
+     *     c. available-brokers: broker3                   => result: NULL
+     *  2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2
+     *     a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2
+     *     b. available-brokers: broker2, broker3          => result: broker2
+     *     c. available-brokers: broker3                   => result: NULL
+     *
+ */ + @Test + public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() + throws IllegalAccessException, BrokerFilterException { + var namespace = "my-tenant/my-ns"; + NamespaceName namespaceName = NamespaceName.get(namespace); + + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(); + var policies = mock(SimpleResourceAllocationPolicies.class); + + // 1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1 + setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 1); + IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + // a. available-brokers: broker1, broker2, broker3 => result: broker1 + Map result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker1")); + + // b. available-brokers: broker2, broker3 => result: broker2 + result = filter.filter(new HashMap<>(Map.of( + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker2")); + + // c. available-brokers: broker3 => result: NULL + result = filter.filter(new HashMap<>(Map.of( + "broker3", getLookupData())), namespaceName, getContext()); + assertTrue(result.isEmpty()); + + // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 + setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 2); + + // a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2 + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker1", "broker2")); + + // b. available-brokers: broker2, broker3 => result: broker2 + result = filter.filter(new HashMap<>(Map.of( + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker2")); + + // c. available-brokers: broker3 => result: NULL + result = filter.filter(new HashMap<>(Map.of( + "broker3", getLookupData())), namespaceName, getContext()); + assertTrue(result.isEmpty()); + } + + @Test + public void testFilterWithPersistentOrNonPersistentDisabled() + throws IllegalAccessException, BrokerFilterException { + var namespace = "my-tenant/my-ns"; + NamespaceName namespaceName = NamespaceName.get(namespace); + NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); + doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); + doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); + + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(); + + var policies = mock(SimpleResourceAllocationPolicies.class); + doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(true).when(policies).isSharedBroker(any()); + IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + Map result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + + + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(true, false), + "broker2", getLookupData(true, false), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker3")); + + doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); + + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(false, true), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker2", "broker3")); + } + + private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, + NamespaceName namespaceName, + Set primary, + Set secondary, + Set shared, + int min_limit) { + reset(policies); + doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(false).when(policies).isPrimaryBroker(eq(namespaceName), any()); + doReturn(false).when(policies).isSecondaryBroker(eq(namespaceName), any()); + doReturn(false).when(policies).isSharedBroker(any()); + + primary.forEach(broker -> { + doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); + }); + + secondary.forEach(broker -> { + doReturn(true).when(policies).isSecondaryBroker(eq(namespaceName), eq(broker)); + }); + + shared.forEach(broker -> { + doReturn(true).when(policies).isSharedBroker(eq(broker)); + }); + + doAnswer(invocationOnMock -> { + Integer totalPrimaryCandidates = invocationOnMock.getArgument(1, Integer.class); + return totalPrimaryCandidates < min_limit; + }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt()); + } + + public BrokerLookupData getLookupData() { + return getLookupData(true, true); + } + + public BrokerLookupData getLookupData(boolean persistentTopicsEnabled, + boolean nonPersistentTopicsEnabled) { + String webServiceUrl = "http://localhost:8080"; + String webServiceUrlTls = "https://localhoss:8081"; + String pulsarServiceUrl = "pulsar://localhost:6650"; + String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, + persistentTopicsEnabled, nonPersistentTopicsEnabled, "3.0.0"); + } + + public LoadManagerContext getContext() { + LoadManagerContext mockContext = mock(LoadManagerContext.class); + doReturn(new ServiceConfiguration()).when(mockContext).brokerConfiguration(); + return mockContext; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index 4c3255341b778..da13a9526a881 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -58,7 +58,7 @@ public void test() throws IllegalAccessException, BrokerFilterException { "broker3", getLookupData(), "broker4", getLookupData() ); - Map result = filter.filter(new HashMap<>(originalBrokers), context); + Map result = filter.filter(new HashMap<>(originalBrokers), null, context); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index d36c79d60ed4e..cafd8f0ea7a4c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -41,7 +41,7 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test public void testFilterEmptyBrokerList() throws BrokerFilterException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), getContext()); + Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()); assertTrue(result.isEmpty()); } @@ -58,7 +58,7 @@ public void testDisabledFilter() throws BrokerFilterException { ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, context); + Map result = brokerVersionFilter.filter(brokers, null, context); assertEquals(result, originalBrokers); } @@ -71,7 +71,8 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.1") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + Map result = brokerVersionFilter.filter( + new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), "localhost:6652", getLookupData("2.10.1"), @@ -84,7 +85,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -98,7 +99,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); @@ -111,6 +112,6 @@ public void testInvalidVersionString() throws BrokerFilterException { "localhost:6650", getLookupData("xxx") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 709a1113f35e4..a668d85f0071c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -28,9 +28,15 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -42,24 +48,37 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.commons.math3.stat.descriptive.moment.Mean; import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundles; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; @@ -145,6 +164,19 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) { return topKBundles.getLoadData(); } + public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace, + int load1, + int load2) { + var namespaceBundleStats1 = new NamespaceBundleStats(); + namespaceBundleStats1.msgThroughputOut = load1 * 1e6; + var namespaceBundleStats2 = new NamespaceBundleStats(); + namespaceBundleStats2.msgThroughputOut = load2 * 1e6; + var topKBundles = new TopKBundles(); + topKBundles.update(Map.of(namespace + "/0x00000000_0x7FFFFFF", namespaceBundleStats1, + namespace + "/0x7FFFFFF_0xFFFFFFF", namespaceBundleStats2), 2); + return topKBundles.getLoadData(); + } + public LoadManagerContext getContext(){ var ctx = mock(LoadManagerContext.class); var conf = new ServiceConfiguration(); @@ -236,6 +268,10 @@ public int size() { doReturn(conf).when(ctx).brokerConfiguration(); doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore(); doReturn(topBundleLoadDataStore).when(ctx).topBundleLoadDataStore(); + var brokerRegister = mock(BrokerRegistry.class); + doReturn(brokerRegister).when(ctx).brokerRegistry(); + BrokerRegistry registry = ctx.brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of())).when(registry).getAvailableBrokerLookupDataAsync(); return ctx; } @@ -350,33 +386,187 @@ public void testRecentlyUnloadedBundles() { } @Test - public void testBundlesWithIsolationPolicies() throws IllegalAccessException { - var pulsar = mock(PulsarService.class); + public void testGetAvailableBrokersFailed() { + TransferShedder transferShedder = new TransferShedder(); + var ctx = setupContext(); + BrokerRegistry registry = ctx.brokerRegistry(); + doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync(); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + var expected = new UnloadDecision(); + expected.setLabel(Skip); + expected.skip(Unknown); + expected.setLoadAvg(setupLoadAvg); + expected.setLoadStd(setupLoadStd); + assertEquals(res, expected); + } + + @Test(timeOut = 30 * 1000) + public void testBundlesWithIsolationPolicies() throws IllegalAccessException, MetadataStoreException { + var pulsar = getMockPulsar(); + TransferShedder transferShedder = new TransferShedder(pulsar); + + var pulsarResourcesMock = mock(PulsarResources.class); + var localPoliciesResourcesMock = mock(LocalPoliciesResources.class); + doReturn(pulsarResourcesMock).when(pulsar).getPulsarResources(); + doReturn(localPoliciesResourcesMock).when(pulsarResourcesMock).getLocalPolicies(); + doReturn(Optional.empty()).when(localPoliciesResourcesMock).getLocalPolicies(any()); + var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); - doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true); + IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy); + FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + // Test transfer to a has isolation policies broker. + setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", + Set.of("broker5"), Set.of(), Set.of(), 1); var ctx = setupContext(); + BrokerRegistry registry = ctx.brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData(), + "broker4", getLookupData(), + "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync(); + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA/0x00000000_0xFFFFFFF", 1, 3)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB/0x00000000_0xFFFFFFF", 2, 8)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC/0x00000000_0xFFFFFFF", 6, 10)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD/0x00000000_0xFFFFFFF", 10, 20)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE/0x00000000_0xFFFFFFF", 70, 20)); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8)); + topBundlesLoadDataStore.pushAsync("broker3", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10)); + topBundlesLoadDataStore.pushAsync("broker4", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20)); + topBundlesLoadDataStore.pushAsync("broker5", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(NoBundles); + var unloads = expected.getUnloads(); + unloads.put("broker4", + new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.of("broker2"))); + expected.setLabel(Success); + expected.setReason(Overloaded); + expected.setLoadAvg(setupLoadAvg); + expected.setLoadStd(setupLoadStd); + assertEquals(res, expected); + + // Test unload a has isolation policies broker. + + setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", + Set.of("broker5"), Set.of(), Set.of(), 1); + ctx = setupContext(); + registry = ctx.brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData(), + "broker4", getLookupData(), + "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync(); + + ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false); + + topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8)); + topBundlesLoadDataStore.pushAsync("broker3", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10)); + topBundlesLoadDataStore.pushAsync("broker4", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20)); + topBundlesLoadDataStore.pushAsync("broker5", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20)); + res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + expected = new UnloadDecision(); + unloads = expected.getUnloads(); + unloads.put("broker4", + new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.empty())); + expected.setLabel(Success); + expected.setReason(Overloaded); expected.setLoadAvg(setupLoadAvg); expected.setLoadStd(setupLoadStd); assertEquals(res, expected); } + public BrokerLookupData getLookupData() { + String webServiceUrl = "http://localhost:8080"; + String webServiceUrlTls = "https://localhoss:8081"; + String pulsarServiceUrl = "pulsar://localhost:6650"; + String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, + true, true, "3.0.0"); + } + + private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, + String namespace, + Set primary, + Set secondary, + Set shared, + int min_limit) { + reset(policies); + NamespaceName namespaceName = NamespaceName.get(namespace); + NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); + doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); + doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); + doReturn(false).when(policies).areIsolationPoliciesPresent(any()); + doReturn(false).when(policies).isPrimaryBroker(any(), any()); + doReturn(false).when(policies).isSecondaryBroker(any(), any()); + doReturn(true).when(policies).isSharedBroker(any()); + + doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + + primary.forEach(broker -> { + doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); + }); + + secondary.forEach(broker -> { + doReturn(true).when(policies).isSecondaryBroker(eq(namespaceName), eq(broker)); + }); + + shared.forEach(broker -> { + doReturn(true).when(policies).isSharedBroker(eq(broker)); + }); + + doAnswer(invocationOnMock -> { + Integer totalPrimaryCandidates = invocationOnMock.getArgument(1, Integer.class); + return totalPrimaryCandidates < min_limit; + }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt()); + } + + private PulsarService getMockPulsar() { + var pulsar = mock(PulsarService.class); + var namespaceService = mock(NamespaceService.class); + doReturn(namespaceService).when(pulsar).getNamespaceService(); + NamespaceBundleFactory factory = mock(NamespaceBundleFactory.class); + doReturn(factory).when(namespaceService).getNamespaceBundleFactory(); + doAnswer(answer -> { + String namespace = answer.getArgument(0, String.class); + String bundleRange = answer.getArgument(1, String.class); + String[] boundaries = bundleRange.split("_"); + Long lowerEndpoint = Long.decode(boundaries[0]); + Long upperEndpoint = Long.decode(boundaries[1]); + Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint, + (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); + return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory); + }).when(factory).getBundle(anyString(), anyString()); + return pulsar; + } + + @Test public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException { - var pulsar = mock(PulsarService.class); + var pulsar = getMockPulsar(); TransferShedder transferShedder = new TransferShedder(pulsar); var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); @@ -392,11 +582,16 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA/0x00000000_0xFFFFFFF", 1, 3)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB/0x00000000_0xFFFFFFF", 2, 8)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC/0x00000000_0xFFFFFFF", 6, 10)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD/0x00000000_0xFFFFFFF", 10, 20)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE/0x00000000_0xFFFFFFF", 70, 20)); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8)); + topBundlesLoadDataStore.pushAsync("broker3", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10)); + topBundlesLoadDataStore.pushAsync("broker4", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20)); + topBundlesLoadDataStore.pushAsync("broker5", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new UnloadDecision(); From 44993ce98280280a2cc9327faa0f1026acf742b3 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Sun, 12 Mar 2023 14:39:30 +0900 Subject: [PATCH 062/404] [fix][broker] Fix issue where msgRateExpired may not refresh forever (#19759) --- .../persistent/PersistentReplicator.java | 2 + .../service/persistent/PersistentTopic.java | 1 + .../api/SimpleProducerConsumerStatTest.java | 41 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 2d341016f089e..f38bcc71582a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -558,6 +558,8 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { public void updateRates() { msgOut.calculateRate(); msgExpired.calculateRate(); + expiryMonitor.updateRates(); + stats.msgRateOut = msgOut.getRate(); stats.msgThroughputOut = msgOut.getValueRate(); stats.msgRateExpired = msgExpired.getRate() + expiryMonitor.getMessageExpiryRate(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 79717ed57303f..b3b6526eea54d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1984,6 +1984,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats // Populate subscription specific stats here topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog(true)); + subscription.getExpiryMonitor().updateRates(); topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate()); topicStatsStream.writePair("msgRateOut", subMsgRateOut); topicStatsStream.writePair("messageAckRate", subMsgAckRate); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java index 2fa3ed9ad01c7..89e6c684ee17f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java @@ -520,4 +520,45 @@ public void testPartitionTopicStats() throws Exception { log.info("-- Exiting {} test --", methodName); } + + @Test + public void testMsgRateExpired() throws Exception { + log.info("-- Starting {} test --", methodName); + + String topicName = "persistent://my-property/tp1/my-ns/" + methodName; + String subName = "my-sub"; + admin.topics().createSubscription(topicName, subName, MessageId.latest); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(false) + .create(); + + int numMessages = 100; + for (int i = 0; i < numMessages; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Thread.sleep(2000); + admin.topics().expireMessages(topicName, subName, 1); + pulsar.getBrokerService().updateRates(); + + Awaitility.await().ignoreExceptions().timeout(5, TimeUnit.SECONDS) + .until(() -> admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired() > 0.001); + + Thread.sleep(2000); + pulsar.getBrokerService().updateRates(); + + Awaitility.await().ignoreExceptions().timeout(5, TimeUnit.SECONDS) + .until(() -> admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired() < 0.001); + + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired(), 0.0, + 0.001); + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getTotalMsgExpired(), + numMessages); + + log.info("-- Exiting {} test --", methodName); + } } From eb3169937e93d87d7587331aab18077849e14ca7 Mon Sep 17 00:00:00 2001 From: ran Date: Sun, 12 Mar 2023 15:16:30 +0800 Subject: [PATCH 063/404] [fix][admin] Filter pending ack topic while deleting the namespace (#19719) ### Motivation A transaction system topic not found exception may occur while deleting the namespace. **How to happen?** 1. Make sure the topic has a pending ack system topic(`public/default/test-delete-ns-sub__transaction_pending_ack`). 2. Delete the namespace `public/default`. 3. Namespace deletion operation will try to delete the user-created topic `public/default/test-delete-ns` first, at this step, the topic will unsubscribe from all subscriptions, and delete the corresponding pending ack system topic. 4. After the namespace deletion operation delete all user-created topics, it will try to delete all system topics, which contain the pending ack topic `public/default/test-delete-ns-sub__transaction_pending_ack`. 5. The topicNotFound exception occurs. There are two ways to fix this problem. 1. Remove the pending ack topics from the pre-delete topic list. 2. Ignore the topicNotFound exception while deleting the namespace. ~~I think ignoring the exception is better because we don't know if there are other similar topics in the future.~~ After discussion, we decide to filter the pending ack topics while deleting the namespace. ### Modifications Ignore the topicNotFound exception while deleting the namespace. --- .../broker/admin/impl/NamespacesBase.java | 7 +++- .../broker/transaction/TransactionTest.java | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 5be675f7b636c..cffc94b189225 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -249,7 +249,7 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime } else { if (SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { topicPolicy.add(topic); - } else { + } else if (!isDeletedAlongWithUserCreatedTopic(topic)) { allSystemTopics.add(topic); } } @@ -344,6 +344,11 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime }); } + private boolean isDeletedAlongWithUserCreatedTopic(String topic) { + // The transaction pending ack topic will be deleted while topic unsubscribe corresponding subscription. + return topic.endsWith(SystemTopicNames.PENDING_ACK_STORE_SUFFIX); + } + private CompletableFuture internalDeletePartitionedTopicsAsync(List topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index a568db3d9f143..20aeac0ed648f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -76,6 +76,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.pulsar.broker.PulsarService; @@ -114,6 +115,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; @@ -1642,4 +1644,36 @@ public void testEncryptionRequired() throws Exception { .send(); txn.commit(); } + + @Test + public void testDeleteNamespace() throws Exception { + String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5); + String topic = namespace + "/test-delete-ns"; + admin.namespaces().createNamespace(namespace); + try (Producer producer = pulsarClient.newProducer() + .topic(topic) + .create()) { + producer.newMessage().value("test".getBytes()).send(); + + Transaction txn = this.pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + try (Consumer consumer = this.pulsarClient.newConsumer() + .topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("sub") + .subscribe()) { + Message message = consumer.receive(); + consumer.acknowledgeAsync(message.getMessageId(), txn).get(); + } + try (Producer outProducer = this.pulsarClient.newProducer() + .topic(topic + "-out") + .create()) { + outProducer.newMessage(txn).value("output".getBytes()).send(); + } + txn.commit(); + } + admin.namespaces().deleteNamespace(namespace, true); + } + } From 5caca1a995d316b73e3d6a9ba2ab4fdcc02211da Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 13 Mar 2023 09:13:11 +0800 Subject: [PATCH 064/404] [improve][meta] Fix busy-loop causes watcher can't acquire lock. (#19769) --- .../java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java | 3 ++- .../test/java/org/apache/pulsar/metadata/ZKSessionTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index a0247e2231949..1ce01f57d4fbe 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -66,7 +66,8 @@ public ZKSessionWatcher(ZooKeeper zk, Consumer sessionListener) { this.scheduler = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("metadata-store-zk-session-watcher")); this.task = - scheduler.scheduleAtFixedRate(catchingAndLoggingThrowables(this::checkConnectionStatus), tickTimeMillis, + scheduler.scheduleWithFixedDelay( + catchingAndLoggingThrowables(this::checkConnectionStatus), tickTimeMillis, tickTimeMillis, TimeUnit.MILLISECONDS); this.currentStatus = SessionEvent.SessionReestablished; diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java index 1757a6b991c0f..36cb0f132ba58 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java @@ -40,7 +40,7 @@ import org.awaitility.Awaitility; import org.testng.annotations.Test; -@Test(groups = "quarantine") +@Test public class ZKSessionTest extends BaseMetadataStoreTest { @Test From 16a30a7037836641bf9453c7d07b3b6aab80ec26 Mon Sep 17 00:00:00 2001 From: Zhangao Date: Mon, 13 Mar 2023 14:13:23 +0800 Subject: [PATCH 065/404] [fix][broker] Fix index generator is not rollback after entries are failed added. (#19727) Co-authored-by: gavingaozhangmin --- .../mledger/impl/ManagedLedgerImpl.java | 7 +++ .../bookkeeper/mledger/impl/OpAddEntry.java | 1 + .../intercept/ManagedLedgerInterceptor.java | 8 ++++ .../ManagedLedgerInterceptorImpl.java | 9 ++++ .../MangedLedgerInterceptorImplTest.java | 43 +++++++++++++++++++ .../AppendIndexMetadataInterceptor.java | 4 ++ 6 files changed, 72 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 9c05fb7c1047e..8376ee1bb8467 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -866,6 +866,13 @@ protected synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { lastAddEntryTimeMs = System.currentTimeMillis(); } + protected void afterFailedAddEntry(int numOfMessages) { + if (managedLedgerInterceptor == null) { + return; + } + managedLedgerInterceptor.afterFailedAddEntry(numOfMessages); + } + protected boolean beforeAddEntry(OpAddEntry addOperation) { // if no interceptor, just return true to make sure addOperation will be initiate() if (managedLedgerInterceptor == null) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index c6c341fd921d1..c56123c24cac1 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -162,6 +162,7 @@ public void initiateShadowWrite() { public void failed(ManagedLedgerException e) { AddEntryCallback cb = callbackUpdater.getAndSet(this, null); + ml.afterFailedAddEntry(this.getNumberOfMessages()); if (cb != null) { ReferenceCountUtil.release(data); cb.addFailed(e, ctx); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java index 412655594c770..d26a5e15735aa 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java @@ -41,6 +41,14 @@ public interface ManagedLedgerInterceptor { */ OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages); + /** + * Intercept When add entry failed. + * @param numberOfMessages + */ + default void afterFailedAddEntry(int numberOfMessages){ + + } + /** * Intercept when ManagedLedger is initialized. * @param propertiesMap map of properties. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index e3b94ec94a4c1..30713c91907ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -79,6 +79,15 @@ public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { return op; } + @Override + public void afterFailedAddEntry(int numberOfMessages) { + for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { + if (interceptor instanceof AppendIndexMetadataInterceptor) { + ((AppendIndexMetadataInterceptor) interceptor).decreaseWithNumberOfMessages(numberOfMessages); + } + } + } + @Override public void onManagedLedgerPropertiesInitialize(Map propertiesMap) { if (propertiesMap == null || propertiesMap.size() == 0) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java index 0cf0adb5181ec..7d164b68147ab 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java @@ -305,6 +305,49 @@ public void testFindPositionByIndex() throws Exception { ledger.close(); } + @Test + public void testAddEntryFailed() throws Exception { + final int MOCK_BATCH_SIZE = 2; + final String ledgerAndCursorName = "testAddEntryFailed"; + + ManagedLedgerInterceptor interceptor = + new ManagedLedgerInterceptorImpl(getBrokerEntryMetadataInterceptors(), null); + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(2); + config.setManagedLedgerInterceptor(interceptor); + + ByteBuf buffer = Unpooled.wrappedBuffer("message".getBytes()); + ManagedLedger ledger = factory.open(ledgerAndCursorName, config); + + ledger.terminate(); + + ManagedLedgerInterceptorImpl interceptor1 = + (ManagedLedgerInterceptorImpl) ledger.getManagedLedgerInterceptor(); + + CountDownLatch countDownLatch = new CountDownLatch(1); + try { + ledger.asyncAddEntry(buffer, MOCK_BATCH_SIZE, new AsyncCallbacks.AddEntryCallback() { + @Override + public void addComplete(Position position, ByteBuf entryData, Object ctx) { + countDownLatch.countDown(); + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + countDownLatch.countDown(); + } + }, null); + + countDownLatch.await(); + assertEquals(interceptor1.getIndex(), -1); + } finally { + ledger.close(); + factory.shutdown(); + } + + } + @Test public void testBeforeAddEntryWithException() throws Exception { final int MOCK_BATCH_SIZE = 2; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java b/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java index de35cc4498d78..941ae16aef1eb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java @@ -50,4 +50,8 @@ public BrokerEntryMetadata interceptWithNumberOfMessages( public long getIndex() { return indexGenerator.get(); } + + public void decreaseWithNumberOfMessages(int numberOfMessages) { + indexGenerator.addAndGet(-numberOfMessages); + } } From b1a463abec0330d3788b1620e50c634c529148c0 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 13 Mar 2023 21:03:22 +0800 Subject: [PATCH 066/404] [fix][broker] Fix admin api status code compatibility (#19782) --- .../broker/admin/impl/PersistentTopicsBase.java | 2 +- .../pulsar/broker/admin/AdminApi2Test.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 633c4747ee068..035e32542ed71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -420,7 +420,7 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean .thenCompose(partitionedTopicMetadata -> { int currentMetadataPartitions = partitionedTopicMetadata.partitions; if (currentMetadataPartitions <= 0) { - throw new RestException(422 /* Unprocessable entity*/, + throw new RestException(Status.CONFLICT /* Unprocessable entity*/, String.format("Topic %s is not the partitioned topic.", topicName)); } if (expectPartitions < currentMetadataPartitions) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 90be220cfd8f7..fb4f880efff07 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -211,6 +211,23 @@ public Object[][] isV1() { return new Object[][] { { true }, { false } }; } + + /** + * It verifies http error code when updating partitions to ensure compatibility. + */ + @Test + public void testUpdatePartitionsErrorCode() { + final String nonPartitionedTopicName = "non-partitioned-topic-name" + UUID.randomUUID(); + try { + // Update a non-partitioned topic + admin.topics().updatePartitionedTopic(nonPartitionedTopicName, 2); + Assert.fail("Expect conflict exception."); + } catch (PulsarAdminException ex) { + Assert.assertEquals(ex.getStatusCode(), 409 /*Conflict*/); + Assert.assertTrue(ex instanceof PulsarAdminException.ConflictException); + } + } + /** * 
      * It verifies increasing partitions for partitioned-topic.

From 9a85dea53a6d0896cc4ba1ceb1ab11a47d5d65da Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Mon, 13 Mar 2023 18:39:07 -0700
Subject: [PATCH 067/404] [improve][broker] PIP-192 Added SplitScheduler and
 DefaultNamespaceBundleSplitStrategyImpl (#19622)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

We will start raising PRs to implement PIP-192, https://github.com/apache/pulsar/issues/16691

### Modifications

This PR implemented
- SplitScheduler
- DefaultNamespaceBundleSplitStrategyImpl
- SplitManager
-  and their unit test.
---
 .../pulsar/broker/ServiceConfiguration.java   |  24 ++
 .../extensions/ExtensibleLoadManagerImpl.java |  22 +-
 .../extensions/manager/SplitManager.java      | 119 ++++++++
 .../extensions/models/SplitCounter.java       |  46 +--
 .../extensions/models/SplitDecision.java      |   9 -
 .../extensions/scheduler/SplitScheduler.java  | 177 +++++++++++
 ...faultNamespaceBundleSplitStrategyImpl.java | 171 +++++++++++
 .../NamespaceBundleSplitStrategy.java         |   7 +-
 .../ExtensibleLoadManagerImplTest.java        |  31 +-
 .../extensions/manager/SplitManagerTest.java  | 222 ++++++++++++++
 .../scheduler/SplitSchedulerTest.java         | 158 ++++++++++
 ...faultNamespaceBundleSplitStrategyTest.java | 284 ++++++++++++++++++
 12 files changed, 1208 insertions(+), 62 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
 rename pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/{scheduler => strategy}/NamespaceBundleSplitStrategy.java (86%)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ec5b0d4042bcd..3c00e905ac723 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2557,6 +2557,30 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se
                     + "(only used in load balancer extension logics)"
     )
     private double loadBalancerBundleLoadReportPercentage = 10;
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            doc = "Service units'(bundles) split interval. Broker periodically checks whether "
+                    + "some service units(e.g. bundles) should split if they become hot-spots. "
+                    + "(only used in load balancer extension logics)"
+    )
+    private int loadBalancerSplitIntervalMinutes = 1;
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Max number of bundles to split to per cycle. "
+                    + "(only used in load balancer extension logics)"
+    )
+    private int loadBalancerMaxNumberOfBundlesToSplitPerCycle = 10;
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Threshold to the consecutive count of fulfilled split conditions. "
+                    + "If the split scheduler consecutively finds bundles that meet split conditions "
+                    + "many times bigger than this threshold, the scheduler will trigger splits on the bundles "
+                    + "(if the number of bundles is less than loadBalancerNamespaceMaximumBundles). "
+                    + "(only used in load balancer extension logics)"
+    )
+    private int loadBalancerNamespaceBundleSplitConditionThreshold = 5;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 82790f44fcb28..c8ac8e4684506 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -45,16 +45,17 @@
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
-import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler;
+import org.apache.pulsar.broker.loadbalance.extensions.scheduler.SplitScheduler;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.UnloadScheduler;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
@@ -103,7 +104,6 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     @Getter
     private final List brokerFilterPipeline;
-
     /**
      * The load data reporter.
      */
@@ -113,9 +113,12 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private ScheduledFuture brokerLoadDataReportTask;
     private ScheduledFuture topBundlesLoadDataReportTask;
+    private SplitScheduler splitScheduler;
 
     private UnloadManager unloadManager;
 
+    private SplitManager splitManager;
+
     private boolean started = false;
 
     private final AssignCounter assignCounter = new AssignCounter();
@@ -166,7 +169,9 @@ public void start() throws PulsarServerException {
         this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
         this.brokerRegistry.start();
         this.unloadManager = new UnloadManager();
+        this.splitManager = new SplitManager(splitCounter);
         this.serviceUnitStateChannel.listen(unloadManager);
+        this.serviceUnitStateChannel.listen(splitManager);
         this.serviceUnitStateChannel.start();
 
         try {
@@ -184,7 +189,6 @@ public void start() throws PulsarServerException {
                 .brokerLoadDataStore(brokerLoadDataStore)
                 .topBundleLoadDataStore(topBundlesLoadDataStore).build();
 
-
         this.brokerLoadDataReporter =
                 new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore);
 
@@ -216,10 +220,12 @@ public void start() throws PulsarServerException {
                         interval,
                         interval, TimeUnit.MILLISECONDS);
 
-        // TODO: Start bundle split scheduler.
         this.unloadScheduler = new UnloadScheduler(
                 pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel);
         this.unloadScheduler.start();
+        this.splitScheduler = new SplitScheduler(
+                pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
+        this.splitScheduler.start();
         this.started = true;
     }
 
@@ -380,6 +386,7 @@ public void close() throws PulsarServerException {
             this.brokerLoadDataStore.close();
             this.topBundlesLoadDataStore.close();
             this.unloadScheduler.close();
+            this.splitScheduler.close();
         } catch (IOException ex) {
             throw new PulsarServerException(ex);
         } finally {
@@ -411,13 +418,6 @@ private void updateUnloadMetrics(UnloadDecision decision) {
         this.unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress()));
     }
 
-    private void updateSplitMetrics(List decisions) {
-        for (var decision : decisions) {
-            splitCounter.update(decision);
-        }
-        this.splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress()));
-    }
-
     public List getMetrics() {
         List metricsCollection = new ArrayList<>();
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java
new file mode 100644
index 0000000000000..71ebbc92a87db
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+
+/**
+ * Split manager.
+ */
+@Slf4j
+public class SplitManager implements StateChangeListener {
+
+
+    private final Map> inFlightSplitRequests;
+
+    private final SplitCounter counter;
+
+    public SplitManager(SplitCounter splitCounter) {
+        this.inFlightSplitRequests = new ConcurrentHashMap<>();
+        this.counter = splitCounter;
+    }
+
+    private void complete(String serviceUnit, Throwable ex) {
+        inFlightSplitRequests.computeIfPresent(serviceUnit, (__, future) -> {
+            if (!future.isDone()) {
+                if (ex != null) {
+                    future.completeExceptionally(ex);
+                } else {
+                    future.complete(null);
+                }
+            }
+            return null;
+        });
+    }
+
+    public CompletableFuture waitAsync(CompletableFuture eventPubFuture,
+                                             String bundle,
+                                             SplitDecision decision,
+                                             long timeout,
+                                             TimeUnit timeoutUnit) {
+        return eventPubFuture
+                .thenCompose(__ -> inFlightSplitRequests.computeIfAbsent(bundle, ignore -> {
+                    log.info("Published the bundle split event for bundle:{}. "
+                                    + "Waiting the split event to complete. Timeout: {} {}",
+                            bundle, timeout, timeoutUnit);
+                    CompletableFuture future = new CompletableFuture<>();
+                    future.orTimeout(timeout, timeoutUnit).whenComplete((v, ex) -> {
+                        if (ex != null) {
+                            inFlightSplitRequests.remove(bundle);
+                            log.warn("Timed out while waiting for the bundle split event: {}", bundle, ex);
+                        }
+                    });
+                    return future;
+                }))
+                .whenComplete((__, ex) -> {
+                    if (ex != null) {
+                        log.error("Failed the bundle split event for bundle:{}", bundle, ex);
+                        counter.update(Failure, Unknown);
+                    } else {
+                        log.info("Completed the bundle split event for bundle:{}", bundle);
+                        counter.update(decision);
+                    }
+                });
+    }
+
+    @Override
+    public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) {
+        ServiceUnitState state = ServiceUnitStateData.state(data);
+        if (t != null && inFlightSplitRequests.containsKey(serviceUnit)) {
+            this.complete(serviceUnit, t);
+            return;
+        }
+        switch (state) {
+            case Deleted, Owned, Init -> this.complete(serviceUnit, t);
+            default -> {
+                if (log.isDebugEnabled()) {
+                    log.debug("Handling {} for service unit {}", data, serviceUnit);
+                }
+            }
+        }
+    }
+
+    public void close() {
+        inFlightSplitRequests.forEach((bundle, future) -> {
+            if (!future.isDone()) {
+                String msg = String.format("Splitting bundle: %s, but the manager already closed.", bundle);
+                log.warn(msg);
+                future.completeExceptionally(new IllegalStateException(msg));
+            }
+        });
+        inFlightSplitRequests.clear();
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java
index 99406412cee2b..ed72b5f586331 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java
@@ -19,10 +19,8 @@
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions;
@@ -32,7 +30,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.apache.commons.lang3.mutable.MutableLong;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.pulsar.common.stats.Metrics;
 
 /**
@@ -40,23 +38,20 @@
  */
 public class SplitCounter {
 
-    long splitCount = 0;
-
-    final Map> breakdownCounters;
+    private long splitCount = 0;
+    private final Map> breakdownCounters;
+    private volatile long updatedAt = 0;
 
     public SplitCounter() {
         breakdownCounters = Map.of(
                 Success, Map.of(
-                        Topics, new MutableLong(),
-                        Sessions, new MutableLong(),
-                        MsgRate, new MutableLong(),
-                        Bandwidth, new MutableLong(),
-                        Admin, new MutableLong()),
-                Skip, Map.of(
-                        Balanced, new MutableLong()
-                        ),
+                        Topics, new AtomicLong(),
+                        Sessions, new AtomicLong(),
+                        MsgRate, new AtomicLong(),
+                        Bandwidth, new AtomicLong(),
+                        Admin, new AtomicLong()),
                 Failure, Map.of(
-                        Unknown, new MutableLong())
+                        Unknown, new AtomicLong())
         );
     }
 
@@ -64,7 +59,16 @@ public void update(SplitDecision decision) {
         if (decision.label == Success) {
             splitCount++;
         }
-        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment();
+        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void update(SplitDecision.Label label, SplitDecision.Reason reason) {
+        if (label == Success) {
+            splitCount++;
+        }
+        breakdownCounters.get(label).get(reason).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
     }
 
     public List toMetrics(String advertisedBrokerAddress) {
@@ -77,17 +81,18 @@ public List toMetrics(String advertisedBrokerAddress) {
         m.put("brk_lb_bundles_split_total", splitCount);
         metrics.add(m);
 
-        for (Map.Entry> etr
+
+        for (Map.Entry> etr
                 : breakdownCounters.entrySet()) {
             var result = etr.getKey();
-            for (Map.Entry counter : etr.getValue().entrySet()) {
+            for (Map.Entry counter : etr.getValue().entrySet()) {
                 var reason = counter.getKey();
                 var count = counter.getValue();
                 Map breakdownDims = new HashMap<>(dimensions);
                 breakdownDims.put("result", result.toString());
                 breakdownDims.put("reason", reason.toString());
                 Metrics breakdownMetric = Metrics.create(breakdownDims);
-                breakdownMetric.put("brk_lb_bundles_split_breakdown_total", count);
+                breakdownMetric.put("brk_lb_bundles_split_breakdown_total", count.get());
                 metrics.add(breakdownMetric);
             }
         }
@@ -95,4 +100,7 @@ public List toMetrics(String advertisedBrokerAddress) {
         return metrics;
     }
 
+    public long updatedAt() {
+        return updatedAt;
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java
index a3dede50c1cd8..433d21a5a613e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java
@@ -19,9 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
 import lombok.Data;
 
@@ -36,7 +34,6 @@ public class SplitDecision {
 
     public enum Label {
         Success,
-        Skip,
         Failure
     }
 
@@ -46,7 +43,6 @@ public enum Reason {
         MsgRate,
         Bandwidth,
         Admin,
-        Balanced,
         Unknown
     }
 
@@ -62,11 +58,6 @@ public void clear() {
         reason = null;
     }
 
-    public void skip() {
-        label = Skip;
-        reason = Balanced;
-    }
-
     public void succeed(Reason reason) {
         label = Success;
         this.reason = reason;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
new file mode 100644
index 0000000000000..589df80fc5c14
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.strategy.DefaultNamespaceBundleSplitStrategyImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.strategy.NamespaceBundleSplitStrategy;
+import org.apache.pulsar.common.stats.Metrics;
+import org.apache.pulsar.common.util.FutureUtil;
+
+/**
+ * Service Unit(e.g. bundles) Split scheduler.
+ */
+@Slf4j
+public class SplitScheduler implements LoadManagerScheduler {
+
+    private final PulsarService pulsar;
+
+    private final ScheduledExecutorService loadManagerExecutor;
+
+    private final LoadManagerContext context;
+
+    private final ServiceConfiguration conf;
+
+    private final ServiceUnitStateChannel serviceUnitStateChannel;
+
+    private final NamespaceBundleSplitStrategy bundleSplitStrategy;
+
+    private final SplitCounter counter;
+
+    private final SplitManager splitManager;
+
+    private final AtomicReference> splitMetrics;
+
+    private volatile ScheduledFuture task;
+
+    private long counterLastUpdatedAt = 0;
+
+    public SplitScheduler(PulsarService pulsar,
+                          ServiceUnitStateChannel serviceUnitStateChannel,
+                          SplitManager splitManager,
+                          SplitCounter counter,
+                          AtomicReference> splitMetrics,
+                          LoadManagerContext context,
+                          NamespaceBundleSplitStrategy bundleSplitStrategy) {
+        this.pulsar = pulsar;
+        this.loadManagerExecutor = pulsar.getLoadManagerExecutor();
+        this.splitManager = splitManager;
+        this.counter = counter;
+        this.splitMetrics = splitMetrics;
+        this.context = context;
+        this.conf = pulsar.getConfiguration();
+        this.bundleSplitStrategy = bundleSplitStrategy;
+        this.serviceUnitStateChannel = serviceUnitStateChannel;
+    }
+
+    public SplitScheduler(PulsarService pulsar,
+                          ServiceUnitStateChannel serviceUnitStateChannel,
+                          SplitManager splitManager,
+                          SplitCounter counter,
+                          AtomicReference> splitMetrics,
+                          LoadManagerContext context) {
+        this(pulsar, serviceUnitStateChannel, splitManager, counter, splitMetrics, context,
+                new DefaultNamespaceBundleSplitStrategyImpl(counter));
+    }
+
+    @Override
+    public void execute() {
+        boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+        if (debugMode) {
+            log.info("Load balancer enabled: {}, Split enabled: {}.",
+                    conf.isLoadBalancerEnabled(), conf.isLoadBalancerAutoBundleSplitEnabled());
+        }
+
+        if (!isLoadBalancerAutoBundleSplitEnabled()) {
+            if (debugMode) {
+                log.info("The load balancer or load balancer split already disabled. Skipping.");
+            }
+            return;
+        }
+
+        synchronized (bundleSplitStrategy) {
+            final Set decisions = bundleSplitStrategy.findBundlesToSplit(context, pulsar);
+            if (!decisions.isEmpty()) {
+
+                // currently following the unloading timeout
+                var asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs();
+                List> futures = new ArrayList<>();
+                for (SplitDecision decision : decisions) {
+                    if (decision.getLabel() == Success) {
+                        var split = decision.getSplit();
+                        futures.add(
+                                splitManager.waitAsync(
+                                        serviceUnitStateChannel.publishSplitEventAsync(split),
+                                        split.serviceUnit(),
+                                        decision,
+                                        asyncOpTimeoutMs, TimeUnit.MILLISECONDS)
+                        );
+                    }
+                }
+                try {
+                    FutureUtil.waitForAll(futures)
+                            .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
+                } catch (Throwable e) {
+                    log.error("Failed to wait for split events to persist.", e);
+                }
+            } else {
+                if (debugMode) {
+                    log.info("BundleSplitStrategy returned no bundles to split.");
+                }
+            }
+        }
+
+        if (counter.updatedAt() > counterLastUpdatedAt) {
+            splitMetrics.set(counter.toMetrics(pulsar.getAdvertisedAddress()));
+            counterLastUpdatedAt = counter.updatedAt();
+        }
+    }
+
+    @Override
+    public void start() {
+        long interval = TimeUnit.MINUTES
+                .toMillis(conf.getLoadBalancerSplitIntervalMinutes());
+        task = loadManagerExecutor.scheduleAtFixedRate(() -> {
+            try {
+                execute();
+            } catch (Throwable e) {
+                log.error("Failed to run the split job.", e);
+            }
+        }, interval, interval, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void close() {
+        if (task != null) {
+            task.cancel(false);
+            task = null;
+        }
+    }
+
+    private boolean isLoadBalancerAutoBundleSplitEnabled() {
+        return conf.isLoadBalancerEnabled() && conf.isLoadBalancerAutoBundleSplitEnabled();
+    }
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
new file mode 100644
index 0000000000000..e572fd4161bdb
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.strategy;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Topics;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+
+/**
+ * Determines which bundles should be split based on various thresholds.
+ *
+ * Migrate from {@link org.apache.pulsar.broker.loadbalance.impl.BundleSplitterTask}
+ */
+@Slf4j
+public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleSplitStrategy {
+    private final Set decisionCache;
+    private final Map namespaceBundleCount;
+    private final Map bundleHighTrafficFrequency;
+    private final SplitCounter counter;
+
+    public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) {
+        decisionCache = new HashSet<>();
+        namespaceBundleCount = new HashMap<>();
+        bundleHighTrafficFrequency = new HashMap<>();
+        this.counter = counter;
+
+    }
+
+    @Override
+    public Set findBundlesToSplit(LoadManagerContext context, PulsarService pulsar) {
+        decisionCache.clear();
+        namespaceBundleCount.clear();
+        final ServiceConfiguration conf = pulsar.getConfiguration();
+        int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
+        long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
+        long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
+        long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
+        long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
+        long maxSplitCount = conf.getLoadBalancerMaxNumberOfBundlesToSplitPerCycle();
+        long splitConditionThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled();
+
+        Map bundleStatsMap = pulsar.getBrokerService().getBundleStats();
+        NamespaceBundleFactory namespaceBundleFactory =
+                pulsar.getNamespaceService().getNamespaceBundleFactory();
+
+        // clean bundleHighTrafficFrequency
+        bundleHighTrafficFrequency.keySet().retainAll(bundleStatsMap.keySet());
+
+        for (var entry : bundleStatsMap.entrySet()) {
+            final String bundle = entry.getKey();
+            final NamespaceBundleStats stats = entry.getValue();
+            if (stats.topics < 2) {
+                if (debug) {
+                    log.info("The count of topics on the bundle {} is less than 2, skip split!", bundle);
+                }
+                continue;
+            }
+
+            final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+            final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
+            if (!namespaceBundleFactory
+                    .canSplitBundle(namespaceBundleFactory.getBundle(namespaceName, bundleRange))) {
+                if (debug) {
+                    log.info("Can't split the bundle:{}. invalid bundle range:{}. ", bundle, bundleRange);
+                }
+                counter.update(Failure, Unknown);
+                continue;
+            }
+
+            double totalMessageRate = stats.msgRateIn + stats.msgRateOut;
+            double totalMessageThroughput = stats.msgThroughputIn + stats.msgThroughputOut;
+            int totalSessionCount = stats.consumerCount + stats.producerCount;
+            SplitDecision.Reason reason = Unknown;
+            if (stats.topics > maxBundleTopics) {
+                reason = Topics;
+            } else if (maxBundleSessions > 0 && (totalSessionCount > maxBundleSessions)) {
+                reason = Sessions;
+            } else if (totalMessageRate > maxBundleMsgRate) {
+                reason = MsgRate;
+            } else if (totalMessageThroughput > maxBundleBandwidth) {
+                reason = Bandwidth;
+            }
+
+            if (reason != Unknown) {
+                bundleHighTrafficFrequency.put(bundle, bundleHighTrafficFrequency.getOrDefault(bundle, 0) + 1);
+            } else {
+                bundleHighTrafficFrequency.remove(bundle);
+            }
+
+            if (bundleHighTrafficFrequency.getOrDefault(bundle, 0) > splitConditionThreshold) {
+                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+                try {
+                    final int bundleCount = pulsar.getNamespaceService()
+                            .getBundleCount(NamespaceName.get(namespace));
+                    if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0))
+                            < maxBundleCount) {
+                        if (debug) {
+                            log.info("The bundle {} is considered to split. Topics: {}/{}, Sessions: ({}+{})/{}, "
+                                            + "Message Rate: {}/{} (msgs/s), Message Throughput: {}/{} (MB/s)",
+                                    bundle, stats.topics, maxBundleTopics, stats.producerCount, stats.consumerCount,
+                                    maxBundleSessions, totalMessageRate, maxBundleMsgRate,
+                                    totalMessageThroughput / LoadManagerShared.MIBI,
+                                    maxBundleBandwidth / LoadManagerShared.MIBI);
+                        }
+                        var decision = new SplitDecision();
+                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), new HashMap<>()));
+                        decision.succeed(reason);
+                        decisionCache.add(decision);
+                        int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
+                        namespaceBundleCount.put(namespace, bundleNum + 1);
+                        bundleHighTrafficFrequency.remove(bundle);
+                        // Clear namespace bundle-cache
+                        namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName));
+                        if (decisionCache.size() == maxSplitCount) {
+                            if (debug) {
+                                log.info("Too many bundles to split in this split cycle {} / {}. Stop.",
+                                        decisionCache.size(), maxSplitCount);
+                            }
+                            break;
+                        }
+                    } else {
+                        if (debug) {
+                            log.info(
+                                    "Could not split namespace bundle {} because namespace {} has too many bundles:"
+                                            + "{}", bundle, namespace, bundleCount);
+                        }
+                    }
+                } catch (Exception e) {
+                    counter.update(Failure, Unknown);
+                    log.warn("Error while computing bundle splits for namespace {}", namespace, e);
+                }
+            }
+        }
+        return decisionCache;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceBundleSplitStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/NamespaceBundleSplitStrategy.java
similarity index 86%
rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceBundleSplitStrategy.java
rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/NamespaceBundleSplitStrategy.java
index 88bd7f0b08780..14023f1b5b01d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceBundleSplitStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/NamespaceBundleSplitStrategy.java
@@ -16,11 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
+package org.apache.pulsar.broker.loadbalance.extensions.strategy;
 
 import java.util.Set;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
-import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 
 /**
  * Determines which bundles should be split based on various thresholds.
@@ -35,5 +36,5 @@ public interface NamespaceBundleSplitStrategy {
      * @param context The context used for decisions.
      * @return A set of the bundles that should be split.
      */
-    Set findBundlesToSplit(LoadManagerContext context);
+    Set findBundlesToSplit(LoadManagerContext context, PulsarService pulsar);
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 441415a9d35e5..17fcb9952618b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -464,24 +464,16 @@ Unknown, new MutableLong(10))
                     FieldUtils.readDeclaredField(primaryLoadManager, "splitMetrics", true);
             SplitCounter splitCounter = new SplitCounter();
             FieldUtils.writeDeclaredField(splitCounter, "splitCount", 35l, true);
-            FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", new LinkedHashMap<>() {
-                {
-                    put(SplitDecision.Label.Success, new LinkedHashMap<>() {
-                        {
-                            put(Topics, new MutableLong(1));
-                            put(Sessions, new MutableLong(2));
-                            put(MsgRate, new MutableLong(3));
-                            put(Bandwidth, new MutableLong(4));
-                            put(Admin, new MutableLong(5));
-                        }
-                    });
-                    put(SplitDecision.Label.Skip, Map.of(
-                            SplitDecision.Reason.Balanced, new MutableLong(6)
-                    ));
-                    put(SplitDecision.Label.Failure, Map.of(
-                            SplitDecision.Reason.Unknown, new MutableLong(7)));
-                }
-            }, true);
+            FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", Map.of(
+                    SplitDecision.Label.Success, Map.of(
+                            Topics, new AtomicLong(1),
+                            Sessions, new AtomicLong(2),
+                            MsgRate, new AtomicLong(3),
+                            Bandwidth, new AtomicLong(4),
+                            Admin, new AtomicLong(5)),
+                    SplitDecision.Label.Failure, Map.of(
+                            SplitDecision.Reason.Unknown, new AtomicLong(6))
+            ), true);
             splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress()));
         }
 
@@ -556,8 +548,7 @@ SplitDecision.Reason.Balanced, new MutableLong(6)
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=MsgRate, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}]
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}]
-                        dimensions=[{broker=localhost, metric=bundlesSplit, reason=Balanced, result=Skip}], metrics=[{brk_lb_bundles_split_breakdown_total=6}]
-                        dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=7}]
+                        dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=6}]
                         dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}]
                         dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}]
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java
new file mode 100644
index 0000000000000..3287306ab48ba
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class SplitManagerTest {
+    
+    String bundle = "bundle-1";
+
+    String dstBroker = "broker-1";
+
+    @Test
+    public void testEventPubFutureHasException() {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(FutureUtil.failedFuture(new Exception("test")),
+                        bundle, decision, 10, TimeUnit.SECONDS);
+
+        assertTrue(future.isCompletedExceptionally());
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertEquals(ex.getCause().getMessage(), "test");
+        }
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, Unknown);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+    }
+
+    @Test
+    public void testTimeout() throws IllegalAccessException {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 3, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getCause() instanceof TimeoutException);
+        }
+
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, Unknown);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+    }
+
+    @Test
+    public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var counterExpected = new SplitCounter();
+        var decision = new SplitDecision();
+        decision.succeed(Sessions);
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 5, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Assigning, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Splitting, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Free, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Deleted, dstBroker, VERSION_ID_INIT), null);
+        counterExpected.update(SplitDecision.Label.Success, Sessions);
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+
+        // Success with Init state.
+        future = manager.waitAsync(CompletableFuture.completedFuture(null),
+                bundle, decision, 5, TimeUnit.SECONDS);
+        inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Init, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        counterExpected.update(SplitDecision.Label.Success, Sessions);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+        future.get();
+
+        // Success with Owned state.
+        future = manager.waitAsync(CompletableFuture.completedFuture(null),
+                bundle, decision, 5, TimeUnit.SECONDS);
+        inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        counterExpected.update(SplitDecision.Label.Success, Sessions);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+        future.get();
+    }
+
+    @Test
+    public void testFailedStage() throws IllegalAccessException {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 5, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, VERSION_ID_INIT),
+                new IllegalStateException("Failed stage."));
+
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getCause() instanceof IllegalStateException);
+            assertEquals(ex.getCause().getMessage(), "Failed stage.");
+        }
+
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, Unknown);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+    }
+
+    @Test
+    public void testClose() throws IllegalAccessException {
+        SplitManager manager = new SplitManager(new SplitCounter());
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 5, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        manager.close();
+        assertEquals(inFlightUnloadRequests.size(), 0);
+
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getCause() instanceof IllegalStateException);
+        }
+    }
+
+    private Map> getinFlightUnloadRequests(SplitManager manager)
+            throws IllegalAccessException {
+        var inFlightUnloadRequest =
+                (Map>) FieldUtils.readField(manager, "inFlightSplitRequests", true);
+
+        return inFlightUnloadRequest;
+    }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
new file mode 100644
index 0000000000000..7988aa413366f
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.strategy.NamespaceBundleSplitStrategy;
+import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.stats.Metrics;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class SplitSchedulerTest {
+
+    PulsarService pulsar;
+    ServiceConfiguration config;
+    NamespaceBundleFactory namespaceBundleFactory;
+    LoadManagerContext context;
+    ServiceUnitStateChannel channel;
+    NamespaceBundleSplitStrategy strategy;
+    String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF";
+    String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF";
+    String broker = "broker-1";
+    SplitDecision decision1;
+    SplitDecision decision2;
+
+    @BeforeMethod
+    public void setUp() {
+
+        config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
+
+        pulsar = mock(PulsarService.class);
+        namespaceBundleFactory = mock(NamespaceBundleFactory.class);
+        context = mock(LoadManagerContext.class);
+        channel = mock(ServiceUnitStateChannel.class);
+        strategy = mock(NamespaceBundleSplitStrategy.class);
+
+        doReturn(config).when(pulsar).getConfiguration();
+        doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
+        doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any());
+
+        decision1 = new SplitDecision();
+        decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+        decision1.succeed(SplitDecision.Reason.MsgRate);
+
+        decision2 = new SplitDecision();
+        decision2.setSplit(new Split(bundle2, broker, new HashMap<>()));
+        decision2.succeed(SplitDecision.Reason.Sessions);
+        Set decisions = Set.of(decision1, decision2);
+        doReturn(decisions).when(strategy).findBundlesToSplit(any(), any());
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testExecuteSuccess() {
+        AtomicReference> reference = new AtomicReference();
+        SplitCounter counter = new SplitCounter();
+        SplitManager manager = mock(SplitManager.class);
+        SplitScheduler scheduler = new SplitScheduler(pulsar, channel, manager, counter, reference, context, strategy);
+        doAnswer((invocation)->{
+            var decision = invocation.getArgument(2, SplitDecision.class);
+            counter.update(decision);
+            return CompletableFuture.completedFuture(null);
+        }).when(manager).waitAsync(any(), any(), any(), anyLong(), any());
+        scheduler.execute();
+
+        var counterExpected = new SplitCounter();
+        counterExpected.update(decision1);
+        counterExpected.update(decision2);
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision1.getSplit()));
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision2.getSplit()));
+
+        assertEquals(reference.get().toString(), counterExpected.toMetrics(pulsar.getAdvertisedAddress()).toString());
+
+        // Test empty splits.
+        Set emptyUnload = Set.of();
+        doReturn(emptyUnload).when(strategy).findBundlesToSplit(any(), any());
+
+        scheduler.execute();
+        verify(channel, times(2)).publishSplitEventAsync(any());
+        assertEquals(reference.get().toString(), counterExpected.toMetrics(pulsar.getAdvertisedAddress()).toString());
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testExecuteFailure() {
+        AtomicReference> reference = new AtomicReference();
+        SplitCounter counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        SplitScheduler scheduler = new SplitScheduler(pulsar, channel, manager, counter, reference, context, strategy);
+        doReturn(CompletableFuture.failedFuture(new RuntimeException())).when(channel).publishSplitEventAsync(any());
+
+        scheduler.execute();
+
+
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, SplitDecision.Reason.Unknown);
+        counterExpected.update(SplitDecision.Label.Failure, SplitDecision.Reason.Unknown);
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision1.getSplit()));
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision2.getSplit()));
+
+        assertEquals(reference.get().toString(), counterExpected.toMetrics(pulsar.getAdvertisedAddress()).toString());
+    }
+
+
+    @Test(timeOut = 30 * 1000)
+    public void testDisableLoadBalancer() {
+
+        config.setLoadBalancerEnabled(false);
+        SplitScheduler scheduler = new SplitScheduler(pulsar, channel, null, null, null, context, strategy);
+
+        scheduler.execute();
+
+        verify(strategy, times(0)).findBundlesToSplit(any(), any());
+
+        config.setLoadBalancerEnabled(true);
+        config.setLoadBalancerAutoBundleSplitEnabled(false);
+        scheduler.execute();
+
+        verify(strategy, times(0)).findBundlesToSplit(any(), any());
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
new file mode 100644
index 0000000000000..71606bb85a3fe
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.strategy;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.namespace.NamespaceService;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.service.PulsarStats;
+import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class DefaultNamespaceBundleSplitStrategyTest {
+
+    PulsarService pulsar;
+    BrokerService brokerService;
+    PulsarStats pulsarStats;
+    Map bundleStats;
+    ServiceConfiguration config;
+    NamespaceBundleFactory namespaceBundleFactory;
+    NamespaceService namespaceService;
+
+    LoadManagerContext loadManagerContext;
+
+    BrokerRegistry brokerRegistry;
+
+    String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF";
+    String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF";
+
+    String broker = "broker-1";
+
+    @BeforeMethod
+    void setup() {
+        config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
+        config.setLoadBalancerNamespaceMaximumBundles(100);
+        config.setLoadBalancerNamespaceBundleMaxTopics(100);
+        config.setLoadBalancerNamespaceBundleMaxSessions(100);
+        config.setLoadBalancerNamespaceBundleMaxMsgRate(100);
+        config.setLoadBalancerNamespaceBundleMaxBandwidthMbytes(100);
+        config.setLoadBalancerMaxNumberOfBundlesToSplitPerCycle(1);
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(3);
+
+        pulsar = mock(PulsarService.class);
+        brokerService = mock(BrokerService.class);
+        pulsarStats = mock(PulsarStats.class);
+        namespaceService = mock(NamespaceService.class);
+        namespaceBundleFactory = mock(NamespaceBundleFactory.class);
+        loadManagerContext = mock(LoadManagerContext.class);
+        brokerRegistry = mock(BrokerRegistry.class);
+
+
+
+        doReturn(brokerService).when(pulsar).getBrokerService();
+        doReturn(config).when(pulsar).getConfiguration();
+        doReturn(pulsarStats).when(brokerService).getPulsarStats();
+        doReturn(namespaceService).when(pulsar).getNamespaceService();
+        doReturn(namespaceBundleFactory).when(namespaceService).getNamespaceBundleFactory();
+        doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
+        doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry();
+        doReturn(broker).when(brokerRegistry).getBrokerId();
+
+
+        bundleStats = new LinkedHashMap<>();
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.topics = 5;
+        bundleStats.put(bundle1, stats1);
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.topics = 5;
+        bundleStats.put(bundle2, stats2);
+        doReturn(bundleStats).when(brokerService).getBundleStats();
+    }
+
+    public void testNamespaceBundleSplitConditionThreshold() {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        assertEquals(actual.size(), 1);
+    }
+
+
+    public void testNotEnoughTopics() {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        bundleStats.values().forEach(v -> v.topics = 1);
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+    }
+
+    public void testNamespaceMaximumBundles() throws Exception {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        doReturn(config.getLoadBalancerNamespaceMaximumBundles()).when(namespaceService).getBundleCount(any());
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+    }
+
+    public void testEmptyBundleStats() {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        bundleStats.clear();
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+    }
+
+    public void testError() throws Exception {
+        var counter = spy(new SplitCounter());
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        doThrow(new RuntimeException()).when(namespaceService).getBundleCount(any());
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+        verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+    }
+
+    public void testMaxMsgRate() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> {
+            v.msgRateOut = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
+            v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
+        });
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.MsgRate);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.MsgRate);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+    public void testMaxTopics() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> v.topics = config.getLoadBalancerNamespaceBundleMaxTopics() + 1);
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Topics);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Topics);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+    public void testMaxSessions() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> {
+            v.producerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
+            v.consumerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
+        });
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Sessions);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Sessions);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+    public void testMaxBandwidthMbytes() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> {
+            v.msgThroughputOut = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
+            v.msgThroughputIn = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
+        });
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Bandwidth);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Bandwidth);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+}

From 2ddfbfe8ce11f3ff66df53a164570335e573bef4 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 14 Mar 2023 00:42:47 -0700
Subject: [PATCH 068/404] [improve][broker] PIP-192 Supports
 AntiAffinityGroupPolicy (#19708)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

We need to support AntiAffinityGroupPolicy in Load Manager Extension as well.

### Modifications
This PR
- added AntiAffinityGroupPolicyFilter used in the broker assignment logic.
- added AntiAffinityGroupPolicyHelper that is passed to TransferShedder and AntiAffinityGroupPolicyFilter
- modified LoadManagerShared.filterAntiAffinityGroupOwnedBrokers and LoadManagerShared.getAntiAffinityNamespaceOwnedBrokers to accept the bundle ownership data from the Load Manager Extension.
- moved ModularLoadManagerImpl.refreshBrokerToFailureDomainMap(..) to LoadManager.refreshBrokerToFailureDomainMap(..) to reuse it in ExtensibleLoadManagerImpl.
- modified AntiAffinityNamespaceGroupTest to reuse the test cases in AntiAffinityNamespaceGroupExtensionTest
---
 .../extensions/ExtensibleLoadManagerImpl.java |  12 +
 .../channel/ServiceUnitStateChannel.java      |   8 +
 .../channel/ServiceUnitStateChannelImpl.java  |   6 +
 .../filter/AntiAffinityGroupPolicyFilter.java |  58 ++++
 .../AntiAffinityGroupPolicyHelper.java        |  97 ++++++
 .../extensions/scheduler/TransferShedder.java |  28 +-
 .../loadbalance/impl/LoadManagerShared.java   | 179 +++++++++-
 .../impl/ModularLoadManagerImpl.java          |  43 +--
 .../AntiAffinityNamespaceGroupTest.java       | 310 ++++++++++--------
 ...tiAffinityNamespaceGroupExtensionTest.java | 196 +++++++++++
 .../scheduler/TransferShedderTest.java        | 216 ++++++------
 11 files changed, 841 insertions(+), 312 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index c8ac8e4684506..7895f06fd5088 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -41,6 +41,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter;
@@ -52,6 +53,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler;
@@ -91,6 +93,10 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private ServiceUnitStateChannel serviceUnitStateChannel;
 
+    private AntiAffinityGroupPolicyFilter antiAffinityGroupPolicyFilter;
+
+    private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+
     private LoadDataStore brokerLoadDataStore;
     private LoadDataStore topBundlesLoadDataStore;
 
@@ -137,6 +143,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
                     CompletableFuture>>newBuilder()
             .build();
 
+
     /**
      * Life cycle: Constructor -> initialize -> start -> close.
      */
@@ -173,6 +180,11 @@ public void start() throws PulsarServerException {
         this.serviceUnitStateChannel.listen(unloadManager);
         this.serviceUnitStateChannel.listen(splitManager);
         this.serviceUnitStateChannel.start();
+        this.antiAffinityGroupPolicyHelper =
+                new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel);
+        antiAffinityGroupPolicyHelper.listenFailureDomainUpdate();
+        this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper);
+        this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter);
 
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index dc4d582ddb081..4e92ad791ab63 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -20,7 +20,9 @@
 
 import java.io.Closeable;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
@@ -164,4 +166,10 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     void listen(StateChangeListener listener);
 
+    /**
+     * Returns service unit ownership entry set.
+     * @return a set of service unit ownership entries
+     */
+    Set> getOwnershipEntrySet();
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 5f24e41dda931..f8686c07f05fa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -1317,5 +1317,11 @@ public List getMetrics() {
     @Override
     public void listen(StateChangeListener listener) {
         this.stateChangeListeners.addListener(listener);
+
+    }
+
+    @Override
+    public Set> getOwnershipEntrySet() {
+        return tableview.entrySet();
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
new file mode 100644
index 0000000000000..358f985f83e12
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.filter;
+
+import java.util.Map;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.common.naming.ServiceUnitId;
+
+/**
+ * Filter by anti-affinity-group-policy.
+ */
+public class AntiAffinityGroupPolicyFilter implements BrokerFilter {
+
+    public static final String FILTER_NAME = "broker_anti_affinity_group_filter";
+
+    private final AntiAffinityGroupPolicyHelper helper;
+
+    public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) {
+        this.helper = helper;
+    }
+
+    @Override
+    public Map filter(
+            Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) {
+        helper.filter(brokers, serviceUnitId.toString());
+        return brokers;
+    }
+
+
+    @Override
+    public String name() {
+        return FILTER_NAME;
+    }
+
+    @Override
+    public void initialize(PulsarService pulsar) {
+        return;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
new file mode 100644
index 0000000000000..28acf5fba0ea1
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.policies;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+
+@Slf4j
+public class AntiAffinityGroupPolicyHelper {
+    PulsarService pulsar;
+    Map brokerToFailureDomainMap;
+    ServiceUnitStateChannel channel;
+
+    public AntiAffinityGroupPolicyHelper(PulsarService pulsar,
+                                  ServiceUnitStateChannel channel){
+
+        this.pulsar = pulsar;
+        this.brokerToFailureDomainMap = new HashMap<>();
+        this.channel = channel;
+    }
+
+    public void filter(
+            Map brokers, String bundle) {
+        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle,
+                brokers.keySet(),
+                channel.getOwnershipEntrySet(), brokerToFailureDomainMap);
+    }
+
+    public boolean canUnload(
+            Map brokers,
+            String bundle,
+            String srcBroker,
+            Optional dstBroker) {
+
+
+
+        try {
+            var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
+                    pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
+            if (antiAffinityGroupOptional.isPresent()) {
+
+                // copy to retain the input brokers
+                Map candidates = new HashMap<>(brokers);
+
+                filter(candidates, bundle);
+
+                candidates.remove(srcBroker);
+
+                // unload case
+                if (dstBroker.isEmpty()) {
+                    return !candidates.isEmpty();
+                }
+
+                // transfer case
+                return candidates.containsKey(dstBroker.get());
+            }
+        } catch (MetadataStoreException e) {
+            log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e);
+            return false;
+        }
+
+        return true;
+    }
+
+    public void listenFailureDomainUpdate() {
+        LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap);
+        // register listeners for domain changes
+        pulsar.getPulsarResources().getClusterResources().getFailureDomainResources()
+                .registerListener(__ -> {
+                    pulsar.getLoadManagerExecutor().execute(() ->
+                            LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap));
+                });
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 810fda320af68..9f9582df2cc28 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -32,7 +32,6 @@
 import java.util.concurrent.TimeoutException;
 import lombok.Getter;
 import lombok.experimental.Accessors;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
@@ -41,13 +40,12 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.NamespaceBundle;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -80,6 +78,7 @@ public class TransferShedder implements NamespaceUnloadStrategy {
     private final PulsarService pulsar;
     private final SimpleResourceAllocationPolicies allocationPolicies;
     private final IsolationPoliciesHelper isolationPoliciesHelper;
+    private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
     private final UnloadDecision decision = new UnloadDecision();
 
@@ -88,12 +87,14 @@ public TransferShedder(){
         this.pulsar = null;
         this.allocationPolicies = null;
         this.isolationPoliciesHelper = null;
+        this.antiAffinityGroupPolicyHelper = null;
     }
 
-    public TransferShedder(PulsarService pulsar){
+    public TransferShedder(PulsarService pulsar, AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper) {
         this.pulsar = pulsar;
         this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
+        this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
     }
 
 
@@ -438,32 +439,25 @@ private boolean hasMsgThroughput(LoadManagerContext context, String broker) {
     private boolean isTransferable(LoadManagerContext context,
                                    Map availableBrokers,
                                    String bundle,
-                                   String maxBroker,
-                                   Optional broker) {
+                                   String srcBroker,
+                                   Optional dstBroker) {
         if (pulsar == null || allocationPolicies == null) {
             return true;
         }
         String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
-        NamespaceName namespaceName = NamespaceName.get(namespace);
         final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
         NamespaceBundle namespaceBundle =
                 pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespace, bundleRange);
 
-        if (!canTransferWithIsolationPoliciesToBroker(context, availableBrokers, namespaceBundle, maxBroker, broker)) {
+        if (!canTransferWithIsolationPoliciesToBroker(
+                context, availableBrokers, namespaceBundle, srcBroker, dstBroker)) {
             return false;
         }
 
-        try {
-            var localPoliciesOptional = pulsar
-                    .getPulsarResources().getLocalPolicies().getLocalPolicies(namespaceName);
-            if (localPoliciesOptional.isPresent() && StringUtils.isNotBlank(
-                    localPoliciesOptional.get().namespaceAntiAffinityGroup)) {
-                return false;
-            }
-        } catch (MetadataStoreException e) {
-            log.error("Failed to get localPolicies. Assumes that bundle:{} is not transferable.", bundle, e);
+        if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
             return false;
         }
+
         return true;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
index 2c90b8a4047a2..6818ae03b5280 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -38,13 +39,18 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.BrokerHostUsage;
 import org.apache.pulsar.broker.loadbalance.LoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.resources.ClusterResources;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
+import org.apache.pulsar.common.policies.data.FailureDomainImpl;
 import org.apache.pulsar.common.util.DirectMemoryUtils;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.BrokerData;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
@@ -329,6 +335,18 @@ public static void filterAntiAffinityGroupOwnedBrokers(
         try {
             final Map brokerToAntiAffinityNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(pulsar,
                     namespaceName, brokerToNamespaceToBundleRange).get(30, TimeUnit.SECONDS);
+            filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, brokerToDomainMap,
+                    brokerToAntiAffinityNamespaceCount);
+        } catch (Exception e) {
+            LOG.error("Failed to filter anti-affinity group namespace {}", e.getMessage());
+        }
+    }
+
+    private static void filterAntiAffinityGroupOwnedBrokers(
+            final PulsarService pulsar,
+            final Set candidates,
+            Map brokerToDomainMap,
+            Map brokerToAntiAffinityNamespaceCount) {
             if (brokerToAntiAffinityNamespaceCount == null) {
                 // none of the broker owns anti-affinity-namespace so, none of the broker will be filtered
                 return;
@@ -368,6 +386,23 @@ public static void filterAntiAffinityGroupOwnedBrokers(
                 candidates
                         .removeIf(broker -> brokerToAntiAffinityNamespaceCount.get(broker) != finalLeastNamespaceCount);
             }
+    }
+
+    public static void filterAntiAffinityGroupOwnedBrokers(
+            final PulsarService pulsar, final String assignedBundleName,
+            final Set candidates,
+            Set> bundleOwnershipData,
+            Map brokerToDomainMap) {
+        if (candidates.isEmpty()) {
+            return;
+        }
+        final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName);
+        try {
+            final Map brokerToAntiAffinityNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(
+                    pulsar, namespaceName, bundleOwnershipData)
+                    .get(30, TimeUnit.SECONDS);
+            filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, brokerToDomainMap,
+                    brokerToAntiAffinityNamespaceCount);
         } catch (Exception e) {
             LOG.error("Failed to filter anti-affinity group namespace {}", e.getMessage());
         }
@@ -424,14 +459,13 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
                     brokerToNamespaceToBundleRange) {
 
         CompletableFuture> antiAffinityNsBrokersResult = new CompletableFuture<>();
-
-        pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesAsync(NamespaceName.get(namespaceName))
-                .thenAccept(policies -> {
-            if (!policies.isPresent() || StringUtils.isBlank(policies.get().namespaceAntiAffinityGroup)) {
+        getNamespaceAntiAffinityGroupAsync(pulsar, namespaceName)
+                .thenAccept(antiAffinityGroupOptional -> {
+            if (antiAffinityGroupOptional.isEmpty()) {
                 antiAffinityNsBrokersResult.complete(null);
                 return;
             }
-            final String antiAffinityGroup = policies.get().namespaceAntiAffinityGroup;
+            final String antiAffinityGroup = antiAffinityGroupOptional.get();
             final Map brokerToAntiAffinityNamespaceCount = new ConcurrentHashMap<>();
             final List> futures = new ArrayList<>();
             brokerToNamespaceToBundleRange.forEach((broker, nsToBundleRange) -> {
@@ -442,11 +476,27 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
 
                     CompletableFuture future = new CompletableFuture<>();
                     futures.add(future);
+                    countAntiAffinityNamespaceOwnedBrokers(broker, ns, future,
+                            pulsar, antiAffinityGroup, brokerToAntiAffinityNamespaceCount);
+                });
+            });
+            FutureUtil.waitForAll(futures)
+                    .thenAccept(r -> antiAffinityNsBrokersResult.complete(brokerToAntiAffinityNamespaceCount));
+        }).exceptionally(ex -> {
+            // namespace-policies has not been created yet
+            antiAffinityNsBrokersResult.complete(null);
+            return null;
+        });
+        return antiAffinityNsBrokersResult;
+    }
 
-                    pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesAsync(NamespaceName.get(ns))
-                            .thenAccept(nsPolicies -> {
-                        if (nsPolicies.isPresent()
-                                && antiAffinityGroup.equalsIgnoreCase(nsPolicies.get().namespaceAntiAffinityGroup)) {
+    private static void countAntiAffinityNamespaceOwnedBrokers(
+            String broker, String ns, CompletableFuture future, PulsarService pulsar,
+            String targetAntiAffinityGroup, Map brokerToAntiAffinityNamespaceCount) {
+                    getNamespaceAntiAffinityGroupAsync(pulsar, ns)
+                            .thenAccept(antiAffinityGroupOptional -> {
+                        if (antiAffinityGroupOptional.isPresent()
+                                && targetAntiAffinityGroup.equalsIgnoreCase(antiAffinityGroupOptional.get())) {
                             brokerToAntiAffinityNamespaceCount.compute(broker,
                                     (brokerName, count) -> count == null ? 1 : count + 1);
                         }
@@ -455,8 +505,39 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
                         future.complete(null);
                         return null;
                     });
-                });
-            });
+    }
+
+    public static CompletableFuture> getAntiAffinityNamespaceOwnedBrokers(
+            final PulsarService pulsar, final String namespaceName,
+            Set> bundleOwnershipData) {
+
+        CompletableFuture> antiAffinityNsBrokersResult = new CompletableFuture<>();
+        getNamespaceAntiAffinityGroupAsync(pulsar, namespaceName)
+                .thenAccept(antiAffinityGroupOptional -> {
+            if (antiAffinityGroupOptional.isEmpty()) {
+                antiAffinityNsBrokersResult.complete(null);
+                return;
+            }
+            final String antiAffinityGroup = antiAffinityGroupOptional.get();
+            final Map brokerToAntiAffinityNamespaceCount = new ConcurrentHashMap<>();
+            final List> futures = new ArrayList<>();
+
+            bundleOwnershipData
+                    .forEach(etr -> {
+                        var stateData = etr.getValue();
+                        var bundle = etr.getKey();
+                        if (stateData.state() == ServiceUnitState.Owned
+                                && StringUtils.isNotBlank(stateData.dstBroker())) {
+                            CompletableFuture future = new CompletableFuture<>();
+                            futures.add(future);
+                            countAntiAffinityNamespaceOwnedBrokers
+                                    (stateData.dstBroker(),
+                                            LoadManagerShared.getNamespaceNameFromBundleName(bundle),
+                                            future, pulsar,
+                                            antiAffinityGroup, brokerToAntiAffinityNamespaceCount);
+                        }
+                    });
+
             FutureUtil.waitForAll(futures)
                     .thenAccept(r -> antiAffinityNsBrokersResult.complete(brokerToAntiAffinityNamespaceCount));
         }).exceptionally(ex -> {
@@ -467,6 +548,31 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
         return antiAffinityNsBrokersResult;
     }
 
+    public static CompletableFuture> getNamespaceAntiAffinityGroupAsync(
+            PulsarService pulsar, String namespaceName) {
+        return pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesAsync(NamespaceName.get(namespaceName))
+                .thenApply(localPoliciesOptional -> {
+                    if (localPoliciesOptional.isPresent()
+                            && StringUtils.isNotBlank(localPoliciesOptional.get().namespaceAntiAffinityGroup)) {
+                        return Optional.of(localPoliciesOptional.get().namespaceAntiAffinityGroup);
+                    }
+                    return Optional.empty();
+                });
+    }
+
+
+    public static Optional getNamespaceAntiAffinityGroup(
+            PulsarService pulsar, String namespaceName) throws MetadataStoreException {
+        var localPoliciesOptional =
+                pulsar.getPulsarResources().getLocalPolicies().getLocalPolicies(NamespaceName.get(namespaceName));
+        if (localPoliciesOptional.isPresent()
+                && StringUtils.isNotBlank(localPoliciesOptional.get().namespaceAntiAffinityGroup)) {
+            return Optional.of(localPoliciesOptional.get().namespaceAntiAffinityGroup);
+        }
+        return Optional.empty();
+    }
+
+
     /**
      *
      * It checks if given anti-affinity namespace should be unloaded by broker due to load-shedding. If all the brokers
@@ -492,6 +598,13 @@ public static boolean shouldAntiAffinityNamespaceUnload(
 
         Map brokerNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(pulsar, namespace,
                 brokerToNamespaceToBundleRange).get(10, TimeUnit.SECONDS);
+        return shouldAntiAffinityNamespaceUnload(currentBroker, candidateBrokers, brokerNamespaceCount);
+    }
+
+    private static boolean shouldAntiAffinityNamespaceUnload(
+            String currentBroker,
+            Set candidateBrokers,
+            Map brokerNamespaceCount) {
         if (brokerNamespaceCount != null && !brokerNamespaceCount.isEmpty()) {
             int leastNsCount = Integer.MAX_VALUE;
             int currentBrokerNsCount = 0;
@@ -522,6 +635,18 @@ public static boolean shouldAntiAffinityNamespaceUnload(
         return true;
     }
 
+    public static boolean shouldAntiAffinityNamespaceUnload(
+            String namespace, String bundle, String currentBroker,
+            final PulsarService pulsar,
+            Set> bundleOwnershipData,
+            Set candidateBrokers) throws Exception {
+
+        Map brokerNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(
+                pulsar, namespace, bundleOwnershipData)
+                .get(10, TimeUnit.SECONDS);
+        return shouldAntiAffinityNamespaceUnload(currentBroker, candidateBrokers, brokerNamespaceCount);
+    }
+
     public interface BrokerTopicLoadingPredicate {
         boolean isEnablePersistentTopics(String brokerUrl);
 
@@ -554,4 +679,36 @@ public static void filterBrokersWithLargeTopicCount(Set brokerCandidateC
             brokerCandidateCache.addAll(filteredBrokerCandidates);
         }
     }
+
+    public static void refreshBrokerToFailureDomainMap(PulsarService pulsar,
+                                                       Map brokerToFailureDomainMap) {
+        if (!pulsar.getConfiguration().isFailureDomainsEnabled()) {
+            return;
+        }
+        ClusterResources.FailureDomainResources fdr =
+                pulsar.getPulsarResources().getClusterResources().getFailureDomainResources();
+        String clusterName = pulsar.getConfiguration().getClusterName();
+        try {
+            synchronized (brokerToFailureDomainMap) {
+                Map tempBrokerToFailureDomainMap = new HashMap<>();
+                for (String domainName : fdr.listFailureDomains(clusterName)) {
+                    try {
+                        Optional domain = fdr.getFailureDomain(clusterName, domainName);
+                        if (domain.isPresent()) {
+                            for (String broker : domain.get().brokers) {
+                                tempBrokerToFailureDomainMap.put(broker, domainName);
+                            }
+                        }
+                    } catch (Exception e) {
+                        LOG.warn("Failed to get domain {}", domainName, e);
+                    }
+                }
+                brokerToFailureDomainMap.clear();
+                brokerToFailureDomainMap.putAll(tempBrokerToFailureDomainMap);
+            }
+            LOG.info("Cluster domain refreshed {}", brokerToFailureDomainMap);
+        } catch (Exception e) {
+            LOG.warn("Failed to get domain-list for cluster {}", e.getMessage());
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index 59f9836b30975..f135840d60e59 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -57,7 +57,6 @@
 import org.apache.pulsar.broker.loadbalance.ModularLoadManager;
 import org.apache.pulsar.broker.loadbalance.ModularLoadManagerStrategy;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate;
-import org.apache.pulsar.broker.resources.ClusterResources;
 import org.apache.pulsar.broker.stats.prometheus.metrics.Summary;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.util.ExecutorProvider;
@@ -65,8 +64,6 @@
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
-import org.apache.pulsar.common.policies.data.FailureDomainImpl;
-import org.apache.pulsar.common.policies.data.LocalPolicies;
 import org.apache.pulsar.common.policies.data.ResourceQuota;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -272,11 +269,12 @@ public void initialize(final PulsarService pulsar) {
         policies = new SimpleResourceAllocationPolicies(pulsar);
         filterPipeline.add(new BrokerVersionFilter());
 
-        refreshBrokerToFailureDomainMap();
+        LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap);
         // register listeners for domain changes
         pulsar.getPulsarResources().getClusterResources().getFailureDomainResources()
                 .registerListener(__ -> {
-                    executors.execute(() -> refreshBrokerToFailureDomainMap());
+                    executors.execute(
+                            () -> LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap));
                 });
 
         loadSheddingPipeline.add(createLoadSheddingStrategy());
@@ -713,9 +711,8 @@ public boolean shouldNamespacePoliciesUnload(String namespace, String bundle, St
 
     public boolean shouldAntiAffinityNamespaceUnload(String namespace, String bundle, String currentBroker) {
         try {
-            Optional nsPolicies = pulsar.getPulsarResources().getLocalPolicies()
-                    .getLocalPolicies(NamespaceName.get(namespace));
-            if (!nsPolicies.isPresent() || StringUtils.isBlank(nsPolicies.get().namespaceAntiAffinityGroup)) {
+            var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(pulsar, namespace);
+            if (antiAffinityGroupOptional.isEmpty()) {
                 return true;
             }
 
@@ -1159,36 +1156,6 @@ private void deleteTimeAverageDataFromMetadataStoreAsync(String broker) {
         });
     }
 
-    private void refreshBrokerToFailureDomainMap() {
-        if (!pulsar.getConfiguration().isFailureDomainsEnabled()) {
-            return;
-        }
-        ClusterResources.FailureDomainResources fdr =
-                pulsar.getPulsarResources().getClusterResources().getFailureDomainResources();
-        String clusterName = pulsar.getConfiguration().getClusterName();
-        try {
-            synchronized (brokerToFailureDomainMap) {
-                Map tempBrokerToFailureDomainMap = new HashMap<>();
-                for (String domainName : fdr.listFailureDomains(clusterName)) {
-                    try {
-                        Optional domain = fdr.getFailureDomain(clusterName, domainName);
-                        if (domain.isPresent()) {
-                            for (String broker : domain.get().brokers) {
-                                tempBrokerToFailureDomainMap.put(broker, domainName);
-                            }
-                        }
-                    } catch (Exception e) {
-                        log.warn("Failed to get domain {}", domainName, e);
-                    }
-                }
-                this.brokerToFailureDomainMap = tempBrokerToFailureDomainMap;
-            }
-            log.info("Cluster domain refreshed {}", brokerToFailureDomainMap);
-        } catch (Exception e) {
-            log.warn("Failed to get domain-list for cluster {}", e.getMessage());
-        }
-    }
-
     @Override
     public LocalBrokerData getBrokerLocalData(String broker) {
         String key = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, broker);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
index 869fee487ab19..560cfa9216a02 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
@@ -27,25 +27,25 @@
 import com.google.common.collect.Sets;
 import com.google.common.hash.Hashing;
 import java.lang.reflect.Field;
-import java.net.URL;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
-import org.apache.bookkeeper.util.ZkUtils;
+import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.broker.resources.TenantResources;
+import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
@@ -54,42 +54,33 @@
 import org.apache.pulsar.common.naming.NamespaceBundles;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
-import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.FailureDomain;
+import org.apache.pulsar.common.policies.data.Policies;
+import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet;
-import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.awaitility.Awaitility;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
-public class AntiAffinityNamespaceGroupTest {
-    private LocalBookkeeperEnsemble bkEnsemble;
+public class AntiAffinityNamespaceGroupTest extends MockedPulsarServiceBaseTest {
 
-    private URL url1;
+    private PulsarTestContext additionalPulsarTestContext;
     private PulsarService pulsar1;
     private PulsarAdmin admin1;
-
-    private URL url2;
     private PulsarService pulsar2;
-    private PulsarAdmin admin2;
-
-    private String primaryHost;
-    private String secondaryHost;
-
+    protected String primaryHost;
+    protected String secondaryHost;
     private NamespaceBundleFactory nsFactory;
+    protected Object primaryLoadManager;
+    private Object secondaryLoadManager;
 
-    private ModularLoadManagerImpl primaryLoadManager;
-    private ModularLoadManagerImpl secondaryLoadManager;
 
-    private ExecutorService executor;
+    private PulsarResources resources;
 
     private static Object getField(final Object instance, final String fieldName) throws Exception {
         final Field field = instance.getClass().getDeclaredField(fieldName);
@@ -97,87 +88,94 @@ private static Object getField(final Object instance, final String fieldName) th
         return field.get(instance);
     }
 
-    @BeforeMethod
-    void setup() throws Exception {
-        executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS,
-                new LinkedBlockingQueue());
-        // Start local bookkeeper ensemble
-        bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
-        bkEnsemble.start();
-
-        // Start broker 1
-        ServiceConfiguration config1 = new ServiceConfiguration();
-        config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
-        config1.setClusterName("use");
-        config1.setWebServicePort(Optional.of(0));
-        config1.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort());
-        config1.setBrokerShutdownTimeoutMs(0L);
-        config1.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d));
-        config1.setBrokerServicePort(Optional.of(0));
-        config1.setFailureDomainsEnabled(true);
-        config1.setLoadBalancerEnabled(true);
-        config1.setAdvertisedAddress("localhost");
+    void setupConfigs(ServiceConfiguration conf){
+        conf.setAllowAutoTopicCreation(true);
+        conf.setLoadManagerClassName(getLoadManagerClassName());
+        conf.setFailureDomainsEnabled(true);
+        conf.setLoadBalancerEnabled(true);
         // Don't want overloaded threshold to affect namespace placement
-        config1.setLoadBalancerBrokerOverloadedThresholdPercentage(400);
-        createCluster(bkEnsemble.getZkClient(), config1);
-        pulsar1 = new PulsarService(config1);
-        pulsar1.start();
+        conf.setLoadBalancerBrokerOverloadedThresholdPercentage(400);
+    }
 
+    @BeforeClass
+    @Override
+    public void setup() throws Exception {
+        setupConfigs(conf);
+        super.internalSetup(conf);
+        pulsar1 = pulsar;
         primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
-        url1 = new URL("http://127.0.0.1" + ":" + pulsar1.getListenPortHTTP().get());
-        admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();
-
-        // Start broker 2
-        ServiceConfiguration config2 = new ServiceConfiguration();
-        config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
-        config2.setClusterName("use");
-        config2.setWebServicePort(Optional.of(0));
-        config2.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort());
-        config2.setBrokerShutdownTimeoutMs(0L);
-        config2.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d));
-        config2.setBrokerServicePort(Optional.of(0));
-        config2.setFailureDomainsEnabled(true);
-        config2.setAdvertisedAddress("localhost");
-        // Don't want overloaded threshold to affect namespace placement
-        config2.setLoadBalancerBrokerOverloadedThresholdPercentage(400);
-        pulsar2 = new PulsarService(config2);
-        pulsar2.start();
+        admin1 = admin;
 
+        var config2 = getDefaultConf();
+        setupConfigs(config2);
+        additionalPulsarTestContext = createAdditionalPulsarTestContext(config2);
+        pulsar2 = additionalPulsarTestContext.getPulsarService();
         secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());
 
-        url2 = new URL("http://127.0.0.1" + ":" + config2.getWebServicePort().get());
-        admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();
-
-        primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
-        secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
+        primaryLoadManager = getField(pulsar1.getLoadManager().get(), "loadManager");
+        secondaryLoadManager = getField(pulsar2.getLoadManager().get(), "loadManager");
         nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
 
         Awaitility.await().untilAsserted(() -> {
             assertEquals(pulsar1.getState(), PulsarService.State.Started);
             assertEquals(pulsar2.getState(), PulsarService.State.Started);
         });
+
+        admin1.tenants().createTenant("my-tenant",
+                createDefaultTenantInfo());
+    }
+
+    @Override
+    @AfterClass
+    protected void cleanup() throws Exception {
+        pulsar1 = null;
+        pulsar2.close();
+        super.internalCleanup();
+        this.additionalPulsarTestContext.close();
+    }
+
+    protected void beforePulsarStart(PulsarService pulsar) throws Exception {
+        if (resources == null) {
+            MetadataStoreExtended localStore = pulsar.createLocalMetadataStore(null);
+            MetadataStoreExtended configStore = (MetadataStoreExtended) pulsar.createConfigurationMetadataStore(null);
+            resources = new PulsarResources(localStore, configStore);
+        }
+        this.createNamespaceIfNotExists(resources, NamespaceName.SYSTEM_NAMESPACE.getTenant(),
+                NamespaceName.SYSTEM_NAMESPACE);
     }
 
-    @AfterMethod(alwaysRun = true)
-    void shutdown() throws Exception {
-        executor.shutdownNow();
+    protected void createNamespaceIfNotExists(PulsarResources resources,
+                                              String publicTenant,
+                                              NamespaceName ns) throws Exception {
+        TenantResources tr = resources.getTenantResources();
+        NamespaceResources nsr = resources.getNamespaceResources();
+
+        if (!tr.tenantExists(publicTenant)) {
+            tr.createTenant(publicTenant,
+                    TenantInfo.builder()
+                            .adminRoles(Sets.newHashSet(conf.getSuperUserRoles()))
+                            .allowedClusters(Sets.newHashSet(conf.getClusterName()))
+                            .build());
+        }
 
-        admin1.close();
-        admin2.close();
+        if (!nsr.namespaceExists(ns)) {
+            Policies nsp = new Policies();
+            nsp.replication_clusters = Collections.singleton(conf.getClusterName());
+            nsr.createPolicies(ns, nsp);
+        }
+    }
 
-        pulsar2.close();
-        pulsar1.close();
 
-        bkEnsemble.stop();
+    protected Object getBundleOwnershipData(){
+        return ConcurrentOpenHashMap.>>newBuilder().build();
     }
 
-    private void createCluster(ZooKeeper zk, ServiceConfiguration config) throws Exception {
-        ZkUtils.createFullPathOptimistic(zk, "/admin/clusters/" + config.getClusterName(),
-                ObjectMapperFactory.getMapper().writer().writeValueAsBytes(
-                        ClusterData.builder().serviceUrl("http://" + config.getAdvertisedAddress() + ":" + config.getWebServicePort().get()).build()),
-                Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+
+    protected String getLoadManagerClassName() {
+        return ModularLoadManagerImpl.class.getName();
     }
 
+
     @Test
     public void testClusterDomain() {
 
@@ -209,15 +207,13 @@ public void testClusterDomain() {
     @Test
     public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns";
         final int totalNamespaces = 5;
         final String namespaceAntiAffinityGroup = "my-antiaffinity";
         final String bundle = "/0x00000000_0xffffffff";
         final int totalBrokers = 4;
 
         pulsar1.getConfiguration().setFailureDomainsEnabled(true);
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
 
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
@@ -237,8 +233,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         brokerToDomainMap.put("brokerName-3", "domain-1");
 
         Set candidate = new HashSet<>();
-        ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange =
-                ConcurrentOpenHashMap.>>newBuilder().build();
+        Object brokerToNamespaceToBundleRange = getBundleOwnershipData();
 
         assertEquals(brokers.size(), totalBrokers);
 
@@ -246,7 +241,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
 
         // for namespace-0 all brokers available
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(brokers.size(), totalBrokers);
 
@@ -255,7 +250,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-1 only domain-1 brokers are available as broker-0 already owns namespace-0
         assignedNamespace = namespace + "1" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 2);
         candidate.forEach(broker -> assertEquals(brokerToDomainMap.get(broker), "domain-1"));
@@ -265,7 +260,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-2 only brokers available are : broker-1 and broker-3
         assignedNamespace = namespace + "2" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 2);
         assertTrue(candidate.contains("brokerName-1"));
@@ -276,7 +271,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-3 only brokers available are : broker-3
         assignedNamespace = namespace + "3" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 1);
         assertTrue(candidate.contains("brokerName-3"));
@@ -285,7 +280,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-4 only brokers available are : all 4 brokers
         assignedNamespace = namespace + "4" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 4);
     }
@@ -308,14 +303,11 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
     @Test
     public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns-wo-domain";
         final int totalNamespaces = 5;
-        final String namespaceAntiAffinityGroup = "my-antiaffinity";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-wo-domain";
         final String bundle = "/0x00000000_0xffffffff";
 
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
-
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
             admin1.namespaces().createNamespace(ns);
@@ -324,8 +316,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
 
         Set brokers = new HashSet<>();
         Set candidate = new HashSet<>();
-        ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange =
-                ConcurrentOpenHashMap.>>newBuilder().build();
+        Object brokerToNamespaceToBundleRange = getBundleOwnershipData();
         brokers.add("broker-0");
         brokers.add("broker-1");
         brokers.add("broker-2");
@@ -334,7 +325,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
 
         // all brokers available so, candidate will be all 3 brokers
         candidate.addAll(brokers);
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(brokers.size(), 3);
 
@@ -343,7 +334,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
         candidate.addAll(brokers);
         assignedNamespace = namespace + "1" + bundle;
         // available brokers for ns-1 => broker-1, broker-2
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(candidate.size(), 2);
         assertTrue(candidate.contains("broker-1"));
@@ -354,7 +345,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
         candidate.addAll(brokers);
         // available brokers for ns-2 => broker-2
         assignedNamespace = namespace + "2" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(candidate.size(), 1);
         assertTrue(candidate.contains("broker-2"));
@@ -364,14 +355,19 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
         candidate.addAll(brokers);
         // available brokers for ns-3 => broker-0, broker-1, broker-2
         assignedNamespace = namespace + "3" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(candidate.size(), 3);
     }
 
-    private void selectBrokerForNamespace(
-            ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange,
+    protected void selectBrokerForNamespace(
+            Object ownershipData,
             String broker, String namespace, String assignedBundleName) {
+
+        ConcurrentOpenHashMap>>
+                brokerToNamespaceToBundleRange =
+                (ConcurrentOpenHashMap>>) ownershipData;
         ConcurrentOpenHashSet bundleSet =
                 ConcurrentOpenHashSet.newBuilder().build();
         bundleSet.add(assignedBundleName);
@@ -436,15 +432,19 @@ public void testBrokerSelectionForAntiAffinityGroup() throws Exception {
         });
 
         ServiceUnitId serviceUnit1 = makeBundle(tenant, cluster, "ns1");
-        String selectedBroker1 = primaryLoadManager.selectBrokerForAssignment(serviceUnit1).get();
+        String selectedBroker1 = selectBroker(serviceUnit1, primaryLoadManager);
 
         ServiceUnitId serviceUnit2 = makeBundle(tenant, cluster, "ns2");
-        String selectedBroker2 = primaryLoadManager.selectBrokerForAssignment(serviceUnit2).get();
+        String selectedBroker2 = selectBroker(serviceUnit2, primaryLoadManager);
 
         assertNotEquals(selectedBroker1, selectedBroker2);
 
     }
 
+    protected String selectBroker(ServiceUnitId serviceUnit, Object loadManager) {
+        return ((ModularLoadManager) loadManager).selectBrokerForAssignment(serviceUnit).get();
+    }
+
     /**
      * It verifies that load-shedding task should unload namespace only if there is a broker available which doesn't
      * cause uneven anti-affinity namespace distribution.
@@ -460,14 +460,11 @@ public void testBrokerSelectionForAntiAffinityGroup() throws Exception {
     @Test
     public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns-load-shedding-util";
         final int totalNamespaces = 5;
-        final String namespaceAntiAffinityGroup = "my-antiaffinity";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-load-shedding-util";
         final String bundle = "/0x00000000_0xffffffff";
 
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
-
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
             admin1.namespaces().createNamespace(ns);
@@ -476,8 +473,7 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
 
         Set brokers = new HashSet<>();
         Set candidate = new HashSet<>();
-        ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange =
-                ConcurrentOpenHashMap.>>newBuilder().build();
+        Object brokerToNamespaceToBundleRange = getBundleOwnershipData();
         brokers.add("broker-0");
         brokers.add("broker-1");
         brokers.add("broker-2");
@@ -489,17 +485,17 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
         // add ns-0 to broker-0
         selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-0", namespace + "0", assignedNamespace);
         String currentBroker = "broker-0";
-        boolean shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle,
+        boolean shouldUnload = shouldAntiAffinityNamespaceUnload(namespace + "0", bundle,
                 currentBroker, pulsar1, brokerToNamespaceToBundleRange, candidate);
         assertTrue(shouldUnload);
         // add ns-1 to broker-1
         selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-1", namespace + "1", assignedNamespace);
-        shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
+        shouldUnload = shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
                 pulsar1, brokerToNamespaceToBundleRange, candidate);
         assertTrue(shouldUnload);
         // add ns-2 to broker-2
         selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-2", namespace + "2", assignedNamespace);
-        shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
+        shouldUnload = shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
                 pulsar1, brokerToNamespaceToBundleRange, candidate);
         assertFalse(shouldUnload);
 
@@ -514,14 +510,11 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
     @Test
     public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns-load-shedding";
         final int totalNamespaces = 5;
-        final String namespaceAntiAffinityGroup = "my-antiaffinity";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-load-shedding";
         final String bundle = "0x00000000_0xffffffff";
 
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
-
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
             admin1.namespaces().createNamespace(ns);
@@ -532,22 +525,24 @@ public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {
         PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getSafeWebServiceAddress()).build();
         Producer producer = pulsarClient.newProducer().topic("persistent://" + namespace + "0/my-topic1")
                 .create();
+        pulsar1.getBrokerService().updateRates();
+        verifyLoadSheddingWithAntiAffinityNamespace(namespace + "0", bundle);
+        producer.close();
+    }
+
+    protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, String bundle) {
+
         ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar1
                 .getLoadManager().get()).getLoadManager();
-
-        pulsar1.getBrokerService().updateRates();
         loadManager.updateAll();
-
-        assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, primaryHost));
-        producer.close();
+        assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace, bundle, primaryHost));
     }
 
-    private boolean isLoadManagerUpdatedDomainCache(ModularLoadManagerImpl loadManager) throws Exception {
-        Field mapField = ModularLoadManagerImpl.class.getDeclaredField("brokerToFailureDomainMap");
-        mapField.setAccessible(true);
+    protected boolean isLoadManagerUpdatedDomainCache(Object loadManager) throws Exception {
         @SuppressWarnings("unchecked")
-        Map map = (Map) mapField.get(loadManager);
-        return !map.isEmpty();
+        var brokerToFailureDomainMap = (Map)
+                FieldUtils.readDeclaredField(loadManager, "brokerToFailureDomainMap", true);
+        return !brokerToFailureDomainMap.isEmpty();
     }
 
     private NamespaceBundle makeBundle(final String property, final String cluster, final String namespace) {
@@ -556,4 +551,43 @@ private NamespaceBundle makeBundle(final String property, final String cluster,
                         BoundType.CLOSED));
     }
 
+    private static void filterAntiAffinityGroupOwnedBrokers(
+            PulsarService pulsar,
+            String assignedNamespace,
+            Set brokers,
+            Object ownershipData,
+            Map brokerToDomainMap) {
+        if (ownershipData instanceof Set) {
+            LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, assignedNamespace, brokers,
+                    (Set>) ownershipData, brokerToDomainMap);
+        } else if (ownershipData instanceof ConcurrentOpenHashMap) {
+            LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, assignedNamespace, brokers,
+                    (ConcurrentOpenHashMap>>)
+                            ownershipData, brokerToDomainMap);
+        } else {
+            throw new RuntimeException("Unknown ownershipData class type");
+        }
+    }
+
+    private static boolean shouldAntiAffinityNamespaceUnload(
+            String namespace,
+            String bundle,
+            String currentBroker,
+            PulsarService pulsar,
+            Object ownershipData,
+            Set candidate) throws Exception {
+
+        if (ownershipData instanceof Set) {
+            return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle,
+                    currentBroker, pulsar, (Set>) ownershipData, candidate);
+        } else if (ownershipData instanceof ConcurrentOpenHashMap) {
+            return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle,
+                    currentBroker, pulsar,
+                    (ConcurrentOpenHashMap>>)
+                            ownershipData, candidate);
+        } else {
+            throw new RuntimeException("Unknown ownershipData class type");
+        }
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
new file mode 100644
index 0000000000000..32822c0f5b524
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.loadbalance.AntiAffinityNamespaceGroupTest;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.common.naming.ServiceUnitId;
+import org.testcontainers.shaded.org.apache.commons.lang3.reflect.FieldUtils;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class AntiAffinityNamespaceGroupExtensionTest extends AntiAffinityNamespaceGroupTest {
+
+    final String bundle = "0x00000000_0xffffffff";
+    final String nsSuffix = "-antiaffinity-enabled";
+
+    protected Object getBundleOwnershipData() {
+        return new HashSet>();
+    }
+
+    protected String getLoadManagerClassName() {
+        return ExtensibleLoadManagerImpl.class.getName();
+    }
+
+    protected String selectBroker(ServiceUnitId serviceUnit, Object loadManager) {
+        try {
+            return ((ExtensibleLoadManagerImpl) loadManager).assign(Optional.empty(), serviceUnit).get()
+                    .get().getPulsarServiceUrl();
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected void selectBrokerForNamespace(
+            Object ownershipData,
+            String broker, String namespace, String assignedBundleName) {
+
+        Set> ownershipDataSet =
+                (Set>) ownershipData;
+        ownershipDataSet.add(
+                new AbstractMap.SimpleEntry(
+                        assignedBundleName,
+                        new ServiceUnitStateData(Owned, broker, 1)));
+
+    }
+
+    protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, String bundle) {
+        try {
+            String namespaceBundle = namespace + "/" + bundle;
+            var antiAffinityGroupPolicyHelper =
+                    (AntiAffinityGroupPolicyHelper)
+                            FieldUtils.readDeclaredField(
+                                    primaryLoadManager, "antiAffinityGroupPolicyHelper", true);
+            var brokerRegistry =
+                    (BrokerRegistry)
+                            FieldUtils.readDeclaredField(
+                                    primaryLoadManager, "brokerRegistry", true);
+            var brokers = brokerRegistry
+                    .getAvailableBrokerLookupDataAsync().get(5, TimeUnit.SECONDS);
+            var serviceUnitStateChannel = (ServiceUnitStateChannel)
+                    FieldUtils.readDeclaredField(
+                            primaryLoadManager, "serviceUnitStateChannel", true);
+            var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle)
+                    .get(5, TimeUnit.SECONDS).get();
+            var brokersCopy = new HashMap<>(brokers);
+            brokersCopy.remove(srcBroker);
+            var dstBroker = brokersCopy.entrySet().iterator().next().getKey();
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.empty()));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.of(srcBroker)));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.empty()));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.empty()));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected boolean isLoadManagerUpdatedDomainCache(Object loadManager) throws Exception {
+        @SuppressWarnings("unchecked")
+        var antiAffinityGroupPolicyHelper =
+                (AntiAffinityGroupPolicyHelper)
+                        FieldUtils.readDeclaredField(
+                                loadManager, "antiAffinityGroupPolicyHelper", true);
+        var brokerToFailureDomainMap = (Map)
+                org.apache.commons.lang.reflect.FieldUtils.readDeclaredField(antiAffinityGroupPolicyHelper,
+                        "brokerToFailureDomainMap", true);
+        return !brokerToFailureDomainMap.isEmpty();
+    }
+
+    @Test
+    public void testAntiAffinityGroupPolicyFilter()
+            throws IllegalAccessException, ExecutionException, InterruptedException,
+            TimeoutException, PulsarAdminException, PulsarClientException {
+
+        final String namespace = "my-tenant/test/my-ns-filter";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-filter";
+
+
+        final String antiAffinityEnabledNameSpace = namespace + nsSuffix;
+        admin.namespaces().createNamespace(antiAffinityEnabledNameSpace);
+        admin.namespaces().setNamespaceAntiAffinityGroup(antiAffinityEnabledNameSpace, namespaceAntiAffinityGroup);
+        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar.getSafeWebServiceAddress()).build();
+        @Cleanup
+        Producer producer = pulsarClient.newProducer().topic(
+                        "persistent://" + antiAffinityEnabledNameSpace + "/my-topic1")
+                .create();
+        pulsar.getBrokerService().updateRates();
+        var brokerRegistry =
+                (BrokerRegistry)
+                        FieldUtils.readDeclaredField(
+                                primaryLoadManager, "brokerRegistry", true);
+        var antiAffinityGroupPolicyFilter =
+                (AntiAffinityGroupPolicyFilter)
+                        FieldUtils.readDeclaredField(
+                                primaryLoadManager, "antiAffinityGroupPolicyFilter", true);
+        var context = ((ExtensibleLoadManagerImpl) primaryLoadManager).getContext();
+        var brokers = brokerRegistry
+                .getAvailableBrokerLookupDataAsync().get(5, TimeUnit.SECONDS);
+        ServiceUnitId namespaceBundle = mock(ServiceUnitId.class);
+        doReturn(namespace + "/" + bundle).when(namespaceBundle).toString();
+
+        var expected = new HashMap<>(brokers);
+        var actual = antiAffinityGroupPolicyFilter.filter(
+                brokers, namespaceBundle, context);
+        assertEquals(actual, expected);
+
+        doReturn(antiAffinityEnabledNameSpace + "/" + bundle).when(namespaceBundle).toString();
+        var serviceUnitStateChannel = (ServiceUnitStateChannel)
+                FieldUtils.readDeclaredField(
+                        primaryLoadManager, "serviceUnitStateChannel", true);
+        var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle.toString())
+                .get(5, TimeUnit.SECONDS).get();
+        expected.remove(srcBroker);
+        actual = antiAffinityGroupPolicyFilter.filter(
+                brokers, namespaceBundle, context);
+        assertEquals(actual, expected);
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index a668d85f0071c..47bf1ad2e7ed1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -40,6 +40,8 @@
 import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -50,8 +52,6 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
-import com.google.common.collect.BoundType;
-import com.google.common.collect.Range;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
 import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
@@ -65,11 +65,13 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.LocalPoliciesResources;
+import org.apache.pulsar.broker.resources.NamespaceResources;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
@@ -82,6 +84,7 @@
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 
@@ -89,6 +92,49 @@
 public class TransferShedderTest {
     double setupLoadAvg = 0.36400000000000005;
     double setupLoadStd = 0.3982762860126121;
+
+    PulsarService pulsar;
+    AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+    ServiceConfiguration conf;
+    LocalPoliciesResources localPoliciesResources;
+    String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF";
+    String bundleD2 = "my-tenant/my-namespaceD/0x0FFFFFFF_0xFFFFFFFF";
+    String bundleE1 = "my-tenant/my-namespaceE/0x00000000_0x0FFFFFFF";
+    String bundleE2 = "my-tenant/my-namespaceE/0x0FFFFFFF_0xFFFFFFFF";
+
+    @BeforeMethod
+    public void init() throws MetadataStoreException {
+        pulsar = mock(PulsarService.class);
+        conf = new ServiceConfiguration();
+        doReturn(conf).when(pulsar).getConfiguration();
+
+        var pulsarResources = mock(PulsarResources.class);
+        var namespaceResources = mock(NamespaceResources.class);
+        var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
+        var factory = mock(NamespaceBundleFactory.class);
+        var namespaceService = mock(NamespaceService.class);
+        localPoliciesResources = mock(LocalPoliciesResources.class);
+        antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
+        doReturn(namespaceService).when(pulsar).getNamespaceService();
+        doReturn(pulsarResources).when(pulsar).getPulsarResources();
+        doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
+        doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
+        doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies();
+        doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any());
+        doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any());
+        doReturn(factory).when(namespaceService).getNamespaceBundleFactory();
+        doAnswer(answer -> {
+            String namespace = answer.getArgument(0, String.class);
+            String bundleRange = answer.getArgument(1, String.class);
+            String[] boundaries = bundleRange.split("_");
+            Long lowerEndpoint = Long.decode(boundaries[0]);
+            Long upperEndpoint = Long.decode(boundaries[1]);
+            Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint,
+                    (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
+            return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory);
+        }).when(factory).getBundle(anyString(), anyString());
+        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
+    }
     public LoadManagerContext setupContext(){
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
@@ -101,11 +147,11 @@ public LoadManagerContext setupContext(){
         brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  90));
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 2000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 3000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 4000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("bundleD", 6000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 7000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2000000, 1000000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 3000000, 1000000));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 4000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 6000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 7000000, 2000000));
         return ctx;
     }
 
@@ -151,8 +197,8 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
         var namespaceBundleStats2 = new NamespaceBundleStats();
         namespaceBundleStats2.msgThroughputOut = load2;
         var topKBundles = new TopKBundles();
-        topKBundles.update(Map.of(bundlePrefix + "-1", namespaceBundleStats1,
-                bundlePrefix + "-2", namespaceBundleStats2), 2);
+        topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1,
+                bundlePrefix + "/0x0FFFFFFF_0xFFFFFFFF", namespaceBundleStats2), 2);
         return topKBundles.getLoadData();
     }
 
@@ -160,7 +206,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) {
         var namespaceBundleStats1 = new NamespaceBundleStats();
         namespaceBundleStats1.msgThroughputOut = load1;
         var topKBundles = new TopKBundles();
-        topKBundles.update(Map.of(bundlePrefix + "-1", namespaceBundleStats1), 2);
+        topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1), 2);
         return topKBundles.getLoadData();
     }
 
@@ -265,13 +311,20 @@ public int size() {
                 return map.size();
             }
         };
+
+        BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class),
+                "broker3", mock(BrokerLookupData.class),
+                "broker4", mock(BrokerLookupData.class),
+                "broker5", mock(BrokerLookupData.class)
+        )))
+                .when(brokerRegistry).getAvailableBrokerLookupDataAsync();
         doReturn(conf).when(ctx).brokerConfiguration();
         doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore();
         doReturn(topBundleLoadDataStore).when(ctx).topBundleLoadDataStore();
-        var brokerRegister = mock(BrokerRegistry.class);
-        doReturn(brokerRegister).when(ctx).brokerRegistry();
-        BrokerRegistry registry = ctx.brokerRegistry();
-        doReturn(CompletableFuture.completedFuture(Map.of())).when(registry).getAvailableBrokerLookupDataAsync();
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
         return ctx;
     }
 
@@ -345,9 +398,9 @@ public void testRecentlyUnloadedBrokers() {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
 
         expected.setLabel(Success);
         expected.setReason(Overloaded);
@@ -371,10 +424,10 @@ public void testRecentlyUnloadedBundles() {
         var ctx = setupContext();
         Map recentlyUnloadedBundles = new HashMap<>();
         var now = System.currentTimeMillis();
-        recentlyUnloadedBundles.put("bundleE-1", now);
-        recentlyUnloadedBundles.put("bundleE-2", now);
-        recentlyUnloadedBundles.put("bundleD-1", now);
-        recentlyUnloadedBundles.put("bundleD-2", now);
+        recentlyUnloadedBundles.put(bundleE1, now);
+        recentlyUnloadedBundles.put(bundleE2, now);
+        recentlyUnloadedBundles.put(bundleD1, now);
+        recentlyUnloadedBundles.put(bundleD2, now);
         var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of());
 
         var expected = new UnloadDecision();
@@ -402,16 +455,10 @@ public void testGetAvailableBrokersFailed() {
     }
 
     @Test(timeOut = 30 * 1000)
-    public void testBundlesWithIsolationPolicies() throws IllegalAccessException, MetadataStoreException {
-        var pulsar = getMockPulsar();
+    public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
 
-        TransferShedder transferShedder = new TransferShedder(pulsar);
 
-        var pulsarResourcesMock = mock(PulsarResources.class);
-        var localPoliciesResourcesMock = mock(LocalPoliciesResources.class);
-        doReturn(pulsarResourcesMock).when(pulsar).getPulsarResources();
-        doReturn(localPoliciesResourcesMock).when(pulsarResourcesMock).getLocalPolicies();
-        doReturn(Optional.empty()).when(localPoliciesResourcesMock).getLocalPolicies(any());
+        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
 
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
@@ -423,31 +470,11 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException, Me
         setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
                 Set.of("broker5"), Set.of(), Set.of(), 1);
         var ctx = setupContext();
-        BrokerRegistry registry = ctx.brokerRegistry();
-        doReturn(CompletableFuture.completedFuture(Map.of(
-                "broker1", getLookupData(),
-                "broker2", getLookupData(),
-                "broker3", getLookupData(),
-                "broker4", getLookupData(),
-                "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync();
-
-        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3));
-        topBundlesLoadDataStore.pushAsync("broker2",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8));
-        topBundlesLoadDataStore.pushAsync("broker3",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10));
-        topBundlesLoadDataStore.pushAsync("broker4",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20));
-        topBundlesLoadDataStore.pushAsync("broker5",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker4",
-                new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);
@@ -455,37 +482,12 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException, Me
         assertEquals(res, expected);
 
         // Test unload a has isolation policies broker.
-
-        setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
-                Set.of("broker5"), Set.of(), Set.of(), 1);
-        ctx = setupContext();
-        registry = ctx.brokerRegistry();
-        doReturn(CompletableFuture.completedFuture(Map.of(
-                "broker1", getLookupData(),
-                "broker2", getLookupData(),
-                "broker3", getLookupData(),
-                "broker4", getLookupData(),
-                "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync();
-
         ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false);
-
-        topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3));
-        topBundlesLoadDataStore.pushAsync("broker2",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8));
-        topBundlesLoadDataStore.pushAsync("broker3",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10));
-        topBundlesLoadDataStore.pushAsync("broker4",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20));
-        topBundlesLoadDataStore.pushAsync("broker5",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20));
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
         expected = new UnloadDecision();
         unloads = expected.getUnloads();
         unloads.put("broker4",
-                new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.empty()));
+                new Unload("broker4", bundleD1, Optional.empty()));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);
@@ -567,31 +569,17 @@ private PulsarService getMockPulsar() {
     @Test
     public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
         var pulsar = getMockPulsar();
-        TransferShedder transferShedder = new TransferShedder(pulsar);
+        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
         doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
         FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true);
 
-        var pulsarResourcesMock = mock(PulsarResources.class);
-        var localPoliciesResourcesMock = mock(LocalPoliciesResources.class);
-        doReturn(pulsarResourcesMock).when(pulsar).getPulsarResources();
-        doReturn(localPoliciesResourcesMock).when(pulsarResourcesMock).getLocalPolicies();
         LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup");
-        doReturn(Optional.of(localPolicies)).when(localPoliciesResourcesMock).getLocalPolicies(any());
+        doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any());
 
         var ctx = setupContext();
-        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3));
-        topBundlesLoadDataStore.pushAsync("broker2",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8));
-        topBundlesLoadDataStore.pushAsync("broker3",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10));
-        topBundlesLoadDataStore.pushAsync("broker4",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20));
-        topBundlesLoadDataStore.pushAsync("broker5",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20));
+        doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new UnloadDecision();
@@ -600,6 +588,18 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         expected.setLoadAvg(setupLoadAvg);
         expected.setLoadStd(setupLoadStd);
         assertEquals(res, expected);
+
+        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any());
+        var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected2 = new UnloadDecision();
+        var unloads = expected2.getUnloads();
+        unloads.put("broker5",
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
+        expected2.setLabel(Success);
+        expected2.setReason(Overloaded);
+        expected2.setLoadAvg(setupLoadAvg);
+        expected2.setLoadStd(setupLoadStd);
+        assertEquals(res2, expected2);
     }
 
     @Test
@@ -614,9 +614,9 @@ public void testTargetStd() {
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
 
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 30, 30));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 40, 40));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 50, 50));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 30, 30));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 40, 40));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 50, 50));
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
@@ -633,11 +633,11 @@ public void testSingleTopBundlesLoadData() {
         TransferShedder transferShedder = new TransferShedder();
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 1));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 2));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 6));
-        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("bundleD", 10));
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 70));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 2));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 6));
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 10));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new UnloadDecision();
@@ -661,7 +661,7 @@ public void testTargetStdAfterTransfer() {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(0.26400000000000007);
@@ -687,9 +687,9 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Underloaded);
         expected.setLoadAvg(0.26400000000000007);
@@ -708,9 +708,9 @@ public void testMaxNumberOfTransfersPerShedderCycle() {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);
@@ -723,15 +723,15 @@ public void testRemainingTopBundles() {
         TransferShedder transferShedder = new TransferShedder();
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 3000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);

From 710cea6f6fa2ed11ac67280b000aaa067a767ea0 Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Tue, 14 Mar 2023 16:48:21 +0800
Subject: [PATCH 069/404] [fix][meta] Fix close borrowed executor (#19761)

---
 .../pulsar/metadata/coordination/impl/LeaderElectionImpl.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index ad2a5bef70610..c1121b1309c2c 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -253,8 +253,6 @@ public synchronized CompletableFuture asyncClose() {
 
         internalState = InternalState.Closed;
 
-        executor.shutdownNow();
-
         if (leaderElectionState != LeaderElectionState.Leading) {
             return CompletableFuture.completedFuture(null);
         }

From 4f6b5fee94eee39c7013cf79263d7c31c8c8793b Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 14 Mar 2023 03:53:45 -0500
Subject: [PATCH 070/404] [improve][misc] Add security section to PIP issue
 template (#19803)

---
 .github/ISSUE_TEMPLATE/pip.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/ISSUE_TEMPLATE/pip.yml b/.github/ISSUE_TEMPLATE/pip.yml
index cd9aac33194e2..d494d69947252 100644
--- a/.github/ISSUE_TEMPLATE/pip.yml
+++ b/.github/ISSUE_TEMPLATE/pip.yml
@@ -60,6 +60,15 @@ body:
         This should also serve as documentation for any person that is trying to understand or debug the behavior of a certain feature.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: Security Considerations
+      description: |
+        A detailed description of the security details that ought to be considered for the PIP. This is most relevant for any new HTTP endpoints, new Pulsar Protocol Commands, and new security features. The goal is to describe details like which role will have permission to perform an action.
+
+        If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list.
+    validations:
+      required: true
   - type: textarea
     attributes:
       label: Alternatives

From da3cab5289c41cfe1e064b02ce4794224aeaea04 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Tue, 14 Mar 2023 22:06:21 +0800
Subject: [PATCH 071/404] [improve][broker][PIP-195] Cut off snapshot segment
 according to maxIndexesPerBucketSnapshotSegment (#19706)

---
 conf/broker.conf                              |  7 ++--
 conf/standalone.conf                          | 42 +++++++++++++++++++
 .../pulsar/broker/ServiceConfiguration.java   | 12 +++---
 .../BucketDelayedDeliveryTrackerFactory.java  |  8 +++-
 .../bucket/BucketDelayedDeliveryTracker.java  | 19 ++++++---
 .../broker/delayed/bucket/MutableBucket.java  | 10 +++--
 .../BucketDelayedDeliveryTrackerTest.java     | 41 ++++++++++++------
 .../persistent/BucketDelayedDeliveryTest.java |  1 +
 8 files changed, 109 insertions(+), 31 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 4b7c108be5f11..d52adb254563d 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -571,13 +571,14 @@ delayedDeliveryMinIndexCountPerBucket=50000
 # after reaching the max time step limitation, the snapshot segment will be cut off.
 delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds=300
 
+# The max number of delayed message index in per bucket snapshot segment, -1 means no limitation
+# after reaching the max number limitation, the snapshot segment will be cut off.
+delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
+
 # The max number of delayed message index bucket,
 # after reaching the max buckets limitation, the adjacent buckets will be merged.
 delayedDeliveryMaxNumBuckets=50
 
-# Enable share the delayed message index across subscriptions
-delayedDeliverySharedIndexEnabled=false
-
 # Size of the lookahead window to use when detecting if all the messages in the topic
 # have a fixed delay.
 # Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle
diff --git a/conf/standalone.conf b/conf/standalone.conf
index ed883406883ed..f141946c29f4e 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -1223,3 +1223,45 @@ configurationStoreServers=
 # zookeeper.
 # Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore
 managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1
+
+# Whether to enable the delayed delivery for messages.
+# If disabled, messages will be immediately delivered and there will
+# be no tracking overhead.
+delayedDeliveryEnabled=true
+
+# Class name of the factory that implements the delayed deliver tracker.
+# If value is "org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory",
+# will create bucket based delayed message index tracker.
+delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTrackerFactory
+
+# Control the tick time for when retrying on delayed delivery,
+# affecting the accuracy of the delivery time compared to the scheduled time.
+# Note that this time is used to configure the HashedWheelTimer's tick time for the
+# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory).
+# Default is 1 second.
+delayedDeliveryTickTimeMillis=1000
+
+# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether
+# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
+# time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index
+# for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time
+# has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the
+# delayedDeliveryTickTimeMillis.
+isDelayedDeliveryDeliverAtTimeStrict=false
+
+# The delayed message index bucket min index count.
+# When the index count of the current bucket is more than this value and all message indexes of current ledger
+# have already been added to the tracker we will seal the bucket.
+delayedDeliveryMinIndexCountPerBucket=50000
+
+# The delayed message index bucket time step(in seconds) in per bucket snapshot segment,
+# after reaching the max time step limitation, the snapshot segment will be cut off.
+delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds=300
+
+# The max number of delayed message index in per bucket snapshot segment, -1 means no limitation
+# after reaching the max number limitation, the snapshot segment will be cut off.
+delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
+
+# The max number of delayed message index bucket,
+# after reaching the max buckets limitation, the adjacent buckets will be merged.
+delayedDeliveryMaxNumBuckets=50
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 3c00e905ac723..ff242888ae0b6 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -361,18 +361,20 @@ public class ServiceConfiguration implements PulsarConfiguration {
     private long delayedDeliveryMinIndexCountPerBucket = 50000;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
-            The delayed message index bucket time step(in seconds) in per bucket snapshot segment, \
+            The delayed message index time step(in seconds) in per bucket snapshot segment, \
             after reaching the max time step limitation, the snapshot segment will be cut off.""")
-    private long delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300;
+    private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300;
+
+    @FieldContext(category = CATEGORY_SERVER, doc = """
+            The max number of delayed message index in per bucket snapshot segment, -1 means no limitation\
+            after reaching the max number limitation, the snapshot segment will be cut off.""")
+    private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment = 5000;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
             The max number of delayed message index bucket, \
             after reaching the max buckets limitation, the adjacent buckets will be merged.""")
     private int delayedDeliveryMaxNumBuckets = 50;
 
-    @FieldContext(category = CATEGORY_SERVER, doc = "Enable share the delayed message index across subscriptions")
-    private boolean delayedDeliverySharedIndexEnabled = false;
-
     @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use "
             + "when detecting if all the messages in the topic have a fixed delay. "
             + "Default is 50,000. Setting the lookahead window to 0 will disable the "
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
index ae9cb23ceb922..f0feb8b27d6a1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
@@ -43,7 +43,9 @@ public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrack
 
     private long delayedDeliveryMinIndexCountPerBucket;
 
-    private long delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds;
+    private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds;
+
+    private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment;
 
     @Override
     public void initialize(PulsarService pulsarService) throws Exception {
@@ -58,6 +60,8 @@ public void initialize(PulsarService pulsarService) throws Exception {
         this.delayedDeliveryMaxNumBuckets = config.getDelayedDeliveryMaxNumBuckets();
         this.delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds =
                 config.getDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds();
+        this.delayedDeliveryMaxIndexesPerBucketSnapshotSegment =
+                config.getDelayedDeliveryMaxIndexesPerBucketSnapshotSegment();
     }
 
     @Override
@@ -65,7 +69,7 @@ public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers d
         return new BucketDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, isDelayedDeliveryDeliverAtTimeStrict,
                 bucketSnapshotStorage, delayedDeliveryMinIndexCountPerBucket,
                 TimeUnit.SECONDS.toMillis(delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds),
-                delayedDeliveryMaxNumBuckets);
+                delayedDeliveryMaxIndexesPerBucketSnapshotSegment, delayedDeliveryMaxNumBuckets);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index a77b272297be4..22689cd737b2b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -70,6 +70,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final long timeStepPerBucketSnapshotSegmentInMillis;
 
+    private final int maxIndexesPerBucketSnapshotSegment;
+
     private final int maxNumBuckets;
 
     private long numberDelayedMessages;
@@ -89,9 +91,10 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
                                  BucketSnapshotStorage bucketSnapshotStorage,
                                  long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis,
-                                 int maxNumBuckets) {
+                                 int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) {
         this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict,
-                bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis, maxNumBuckets);
+                bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis,
+                maxIndexesPerBucketSnapshotSegment, maxNumBuckets);
     }
 
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
@@ -99,10 +102,11 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
                                  BucketSnapshotStorage bucketSnapshotStorage,
                                  long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis,
-                                 int maxNumBuckets) {
+                                 int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) {
         super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict);
         this.minIndexCountPerBucket = minIndexCountPerBucket;
         this.timeStepPerBucketSnapshotSegmentInMillis = timeStepPerBucketSnapshotSegmentInMillis;
+        this.maxIndexesPerBucketSnapshotSegment = maxIndexesPerBucketSnapshotSegment;
         this.maxNumBuckets = maxNumBuckets;
         this.sharedBucketPriorityQueue = new TripleLongPriorityQueue();
         this.immutableBuckets = TreeRangeMap.create();
@@ -292,7 +296,9 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
                 && lastMutableBucket.size() >= minIndexCountPerBucket
                 && !lastMutableBucket.isEmpty()) {
             Pair immutableBucketDelayedIndexPair =
-                    lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegmentInMillis,
+                    lastMutableBucket.sealBucketAndAsyncPersistent(
+                            this.timeStepPerBucketSnapshotSegmentInMillis,
+                            this.maxIndexesPerBucketSnapshotSegment,
                             this.sharedBucketPriorityQueue);
             afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
             lastMutableBucket.resetLastMutableBucketRange();
@@ -380,8 +386,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB
                 .thenAccept(combinedDelayedIndexQueue -> {
                     Pair immutableBucketDelayedIndexPair =
                             lastMutableBucket.createImmutableBucketAndAsyncPersistent(
-                                    timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue,
-                                    combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId);
+                                    timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment,
+                                    sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
+                                    bucketB.endLedgerId);
 
                     // Merge bit map to new bucket
                     Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index 40ba8f4c4b593..8187f8f2de158 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -50,13 +50,15 @@ class MutableBucket extends Bucket implements AutoCloseable {
 
     Pair sealBucketAndAsyncPersistent(
             long timeStepPerBucketSnapshotSegment,
+            int maxIndexesPerBucketSnapshotSegment,
             TripleLongPriorityQueue sharedQueue) {
-        return createImmutableBucketAndAsyncPersistent(timeStepPerBucketSnapshotSegment, sharedQueue,
+        return createImmutableBucketAndAsyncPersistent(timeStepPerBucketSnapshotSegment,
+                maxIndexesPerBucketSnapshotSegment, sharedQueue,
                 TripleLongPriorityDelayedIndexQueue.wrap(priorityQueue), startLedgerId, endLedgerId);
     }
 
     Pair createImmutableBucketAndAsyncPersistent(
-            final long timeStepPerBucketSnapshotSegment,
+            final long timeStepPerBucketSnapshotSegment, final int maxIndexesPerBucketSnapshotSegment,
             TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId,
             final long endLedgerId) {
         log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName,
@@ -98,7 +100,9 @@ Pair createImmutableBucketAndAsyncPersistent(
 
             snapshotSegmentBuilder.addIndexes(delayedIndex);
 
-            if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit) {
+            if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit
+                    || (maxIndexesPerBucketSnapshotSegment != -1
+                    && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
                 segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp);
                 currentTimestampUpperLimit = 0;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 0ba9e5f4ca2e7..378ad205bb666 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -85,7 +85,7 @@ public Object[][] provider(Method method) throws Exception {
         return switch (methodName) {
             case "test" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock,
-                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testWithTimer" -> {
                 Timer timer = mock(Timer.class);
@@ -113,39 +113,43 @@ public Object[][] provider(Method method) throws Exception {
 
                 yield new Object[][]{{
                         new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock,
-                                false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50),
+                                false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50),
                         tasks
                 }};
             }
             case "testAddWithinTickTime" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock,
-                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testAddMessageWithStrictDelay" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock,
-                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict", "testRecoverSnapshot" ->
                     new Object[][]{{
                             new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock,
-                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
                     }};
             case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict", "testExistDelayedMessage" ->
                     new Object[][]{{
                             new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock,
-                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
                     }};
             case "testMergeSnapshot", "testWithBkException", "testWithCreateFailDowngrade" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock,
-                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10)
+                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10)
+            }};
+            case "testMaxIndexesPerSegment" -> new Object[][]{{
+                    new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock,
+                            true, bucketSnapshotStorage, 20, TimeUnit.HOURS.toMillis(1), 5, 100)
             }};
             default -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock,
-                            true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50)
+                            true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), -1, 50)
             }};
         };
     }
@@ -196,7 +200,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
         clockTime.set(30 * 10);
 
         tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50);
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,50);
 
         assertFalse(tracker.containsMessage(101, 101));
         assertEquals(tracker.getNumberOfDelayedMessages(), 70);
@@ -268,7 +272,7 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) {
         tracker.close();
 
         tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10);
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
 
         assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue());
 
@@ -318,7 +322,7 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
         tracker.close();
 
         tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10);
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
 
         Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue();
         assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue);
@@ -374,4 +378,17 @@ public void testWithCreateFailDowngrade(BucketDelayedDeliveryTracker tracker) {
             assertEquals(position, PositionImpl.get(i, i));
         }
     }
+
+    @Test(dataProvider = "delayedTracker")
+    public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
+        for (int i = 1; i <= 101; i++) {
+            tracker.addMessage(i, i, i * 10);
+        }
+
+        assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 5);
+
+        tracker.getImmutableBuckets().asMapOfRanges().forEach((k, bucket) -> {
+            assertEquals(bucket.getLastSegmentEntryId(), 4);
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 5d81ba8bc0261..fa846779b08ae 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -32,6 +32,7 @@ public void setup() throws Exception {
         conf.setDelayedDeliveryTrackerFactoryClassName(BucketDelayedDeliveryTrackerFactory.class.getName());
         conf.setDelayedDeliveryMaxNumBuckets(10);
         conf.setDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds(1);
+        conf.setDelayedDeliveryMaxIndexesPerBucketSnapshotSegment(10);
         conf.setDelayedDeliveryMinIndexCountPerBucket(50);
         conf.setManagedLedgerMaxEntriesPerLedger(50);
         conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0);

From 7a11edd4fda75c474b78f1952d13ef104797738a Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Wed, 15 Mar 2023 05:53:41 +0200
Subject: [PATCH 072/404] [fix][doc] Amend PIP voting process description
 (#19810)

---
 wiki/proposals/PIP.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md
index f76c9f0f7a235..e10452b107fd8 100644
--- a/wiki/proposals/PIP.md
+++ b/wiki/proposals/PIP.md
@@ -82,7 +82,7 @@ The process works in the following way:
    the "xxx" number should be chosen to be the next number from the existing PIP 
    issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)).
 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list
-   to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: `. The discussion
+   to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion
    need to happen in the mailing list. Please avoid discussing it using
    GitHub comments in the PIP GitHub issue, as it creates two tracks 
    of feedback.
@@ -90,10 +90,11 @@ The process works in the following way:
    authors to the text of the proposal.
 4. Once some consensus is reached, there will be a vote to formally approve
    the proposal.
-   The vote will be held on the dev@pulsar.apache.org mailing list. Everyone
-   is welcome to vote on the proposal, though it will be considered to be binding
-   only the vote of PMC members.
-   I would be required to have a lazy majority of at least 3 binding +1s votes.
+   The vote will be held on the dev@pulsar.apache.org mailing list, by
+   sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}".
+   Everyone is welcome to vote on the proposal, though only the the vote of the PMC 
+   members will be considered binding.
+   It is required to have a lazy majority of at least 3 binding +1s votes.
    The vote should stay open for at least 48 hours.
 5. When the vote is closed, if the outcome is positive, the state of the
    proposal is updated, and the Pull Requests associated with this proposal can

From 0e96dedf66c1a0961bf5de584ebd0164fec179d9 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 15 Mar 2023 12:02:04 +0800
Subject: [PATCH 073/404] [fix][broker][PIP-195] Don't clean up
 BucketDelayedDeliveryTracker when all consumer disconnect (#19801)

---
 .../pulsar/broker/delayed/bucket/Bucket.java  |  3 +-
 .../bucket/BucketDelayedDeliveryTracker.java  |  4 ++
 .../delayed/bucket/ImmutableBucket.java       | 67 ++++++++++---------
 .../broker/delayed/bucket/MutableBucket.java  |  1 +
 ...PersistentDispatcherMultipleConsumers.java |  8 ++-
 .../BucketDelayedDeliveryTrackerTest.java     | 18 +++++
 .../persistent/BucketDelayedDeliveryTest.java | 59 ++++++++++++++++
 7 files changed, 128 insertions(+), 32 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index 50b5cd12ead07..db864a4e264f5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.delayed.bucket;
 
+import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
 import java.util.HashMap;
 import java.util.List;
@@ -38,7 +39,7 @@
 @AllArgsConstructor
 abstract class Bucket {
 
-    static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket";
+    static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket";
     static final String DELIMITER = "_";
     static final int MaxRetryTimes = 3;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 22689cd737b2b..31fdaa6fb7667 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -76,8 +76,12 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private long numberDelayedMessages;
 
+    @Getter
+    @VisibleForTesting
     private final MutableBucket lastMutableBucket;
 
+    @Getter
+    @VisibleForTesting
     private final TripleLongPriorityQueue sharedBucketPriorityQueue;
 
     @Getter
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index c947e53843a85..c9223efa09243 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -20,6 +20,7 @@
 
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds;
+import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE;
 import com.google.protobuf.ByteString;
 import java.util.Collections;
 import java.util.List;
@@ -167,15 +168,10 @@ CompletableFuture asyncDeleteBucketSnapshot() {
         String bucketKey = bucketKey();
         long bucketId = getAndUpdateBucketId();
         return removeBucketCursorProperty(bucketKey).thenCompose(__ ->
-                executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).whenComplete((___, ex) -> {
-                            if (ex != null) {
-                                log.warn("[{}] Failed to delete bucket snapshot. bucketKey: {}, bucketId: {}",
-                                        dispatcherName, bucketKey, bucketId, ex);
-                            }
-                        }),
+                executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId),
                         BucketSnapshotPersistenceException.class, MaxRetryTimes)).whenComplete((__, ex) -> {
                     if (ex != null) {
-                        log.warn("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
+                        log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
                                 dispatcherName, bucketId, bucketKey, ex);
                     } else {
                         log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
@@ -186,35 +182,46 @@ CompletableFuture asyncDeleteBucketSnapshot() {
 
     void clear(boolean delete) {
         delayedIndexBitMap.clear();
-        getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> {
-            if (delete) {
-                snapshotGenerateFuture.cancel(true);
-                String bucketKey = bucketKey();
-                long bucketId = getAndUpdateBucketId();
-                try {
-                    // Because bucketSnapshotStorage.deleteBucketSnapshot may be use the same thread with clear,
-                    // so we can't block deleteBucketSnapshot when clearing the bucket snapshot.
-                    removeBucketCursorProperty(bucketKey())
-                            .thenApply(__ -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).exceptionally(ex -> {
-                                log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
-                                        bucketId, bucketKey, ex);
-                                return null;
-                            })).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", bucketId, bucketKey, e);
-                    if (e instanceof InterruptedException) {
-                        Thread.currentThread().interrupt();
-                    }
-                    throw new RuntimeException(e);
+        if (delete) {
+            final String bucketKey = bucketKey();
+            try {
+                getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null).thenCompose(__ -> {
+                        if (getSnapshotCreateFuture().isPresent() && getBucketId().isEmpty()) {
+                            log.error("[{}] Can't found bucketId, don't execute delete operate, bucketKey: {}",
+                                    dispatcherName, bucketKey);
+                            return CompletableFuture.completedFuture(null);
+                        }
+                        long bucketId = getAndUpdateBucketId();
+                        return removeBucketCursorProperty(bucketKey()).thenAccept(___ -> {
+                            executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId),
+                            BucketSnapshotPersistenceException.class, MaxRetryTimes)
+                            .whenComplete((____, ex) -> {
+                                if (ex != null) {
+                                    log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
+                                            dispatcherName, bucketId, bucketKey, ex);
+                                } else {
+                                    log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
+                                            dispatcherName, bucketId, bucketKey);
+                                }
+                            });
+                    });
+                }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+            } catch (Exception e) {
+                log.error("Failed to clear bucket snapshot, bucketKey: {}", bucketKey, e);
+                if (e instanceof InterruptedException) {
+                    Thread.currentThread().interrupt();
                 }
-            } else {
+                throw new RuntimeException(e);
+            }
+        } else {
+            getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> {
                 try {
                     snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
                 } catch (Exception e) {
                     log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName,
                             getBucketId(), bucketKey());
                 }
-            }
-        });
+            });
+        }
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index 8187f8f2de158..e743f39e6920d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -175,6 +175,7 @@ void resetLastMutableBucketRange() {
     void clear() {
         this.resetLastMutableBucketRange();
         this.delayedIndexBitMap.clear();
+        this.priorityQueue.clear();
     }
 
     public void close() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 475ba39c51129..059820b1b66c3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -49,6 +49,7 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
+import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
 import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers;
 import org.apache.pulsar.broker.service.BrokerServiceException;
@@ -165,7 +166,12 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce
                 shouldRewindBeforeReadingOrReplaying = false;
             }
             redeliveryMessages.clear();
-            delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::clear);
+            delayedDeliveryTracker.ifPresent(tracker -> {
+                // Don't clean up BucketDelayedDeliveryTracker, otherwise we will lose the bucket snapshot
+                if (tracker instanceof InMemoryDelayedDeliveryTracker) {
+                    tracker.clear();
+                }
+            });
         }
 
         if (isConsumersExceededOnSubscription()) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 378ad205bb666..c834bc7a0b4c5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -391,4 +391,22 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
             assertEquals(bucket.getLastSegmentEntryId(), 4);
         });
     }
+    
+    @Test(dataProvider = "delayedTracker")
+    public void testClear(BucketDelayedDeliveryTracker tracker) {
+      for (int i = 1; i <= 1001; i++) {
+          tracker.addMessage(i, i, i * 10);
+      }
+
+      assertEquals(tracker.getNumberOfDelayedMessages(), 1001);
+      assertTrue(tracker.getImmutableBuckets().asMapOfRanges().size() > 0);
+      assertEquals(tracker.getLastMutableBucket().size(), 1);
+
+      tracker.clear();
+
+      assertEquals(tracker.getNumberOfDelayedMessages(), 0);
+      assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0);
+      assertEquals(tracker.getLastMutableBucket().size(), 0);
+      assertEquals(tracker.getSharedBucketPriorityQueue().size(), 0);
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index fa846779b08ae..292889e8c159a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -18,7 +18,19 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
+import org.apache.pulsar.broker.service.Dispatcher;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -44,4 +56,51 @@ public void setup() throws Exception {
     public void cleanup() throws Exception {
         super.internalCleanup();
     }
+
+    @Test
+    public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exception {
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testDelaysWithAllConsumerDis");
+
+        Consumer c1 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        for (int i = 0; i < 1000; i++) {
+            producer.newMessage()
+                    .value("msg")
+                    .deliverAfter(1, TimeUnit.HOURS)
+                    .send();
+        }
+
+        Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000));
+        List bucketKeys =
+                ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties().keySet().stream()
+                        .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList();
+
+        c1.close();
+
+        // Attach a new consumer. Since there are no consumers connected, this will trigger the cursor rewind
+        @Cleanup
+        Consumer c2 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        Dispatcher dispatcher2 = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        List bucketKeys2 =
+                ((PersistentDispatcherMultipleConsumers) dispatcher2).getCursor().getCursorProperties().keySet().stream()
+                        .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList();
+
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000));
+        Assert.assertEquals(bucketKeys, bucketKeys2);
+    }
 }

From 160a8643a1f6a153106eac9031ac2a9530ac4698 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 15 Mar 2023 13:32:31 +0800
Subject: [PATCH 074/404] [improve][broker][PIP-195] Add topicName and
 cursorName for ledger metadata of bucket snapshot (#19802)

---
 .../mledger/impl/LedgerMetadataUtils.java         | 15 +++++++++++----
 .../bucket/BookkeeperBucketSnapshotStorage.java   |  9 +++++----
 .../pulsar/broker/delayed/bucket/Bucket.java      |  6 +++++-
 .../delayed/bucket/BucketSnapshotStorage.java     |  8 +++++---
 .../pulsar/broker/service/BrokerService.java      |  2 ++
 .../BookkeeperBucketSnapshotStorageTest.java      | 13 ++++++++-----
 .../broker/delayed/MockBucketSnapshotStorage.java |  3 ++-
 .../bucket/BucketDelayedDeliveryTrackerTest.java  |  2 +-
 8 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
index 8571a36584e2b..4ac409a2e9bfe 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
@@ -48,7 +48,9 @@ public final class LedgerMetadataUtils {
     private static final String METADATA_PROPERTY_COMPACTEDTO = "pulsar/compactedTo";
     private static final String METADATA_PROPERTY_SCHEMAID = "pulsar/schemaId";
 
-    private static final String METADATA_PROPERTY_DELAYED_INDEX_BUCKETID = "pulsar/delayedIndexBucketId";
+    private static final String METADATA_PROPERTY_DELAYED_INDEX_BUCKET_KEY = "pulsar/delayedIndexBucketKey";
+    private static final String METADATA_PROPERTY_DELAYED_INDEX_TOPIC = "pulsar/delayedIndexTopic";
+    private static final String METADATA_PROPERTY_DELAYED_INDEX_CURSOR = "pulsar/delayedIndexCursor";
 
     /**
      * Build base metadata for every ManagedLedger.
@@ -108,14 +110,19 @@ public static Map buildMetadataForSchema(String schemaId) {
     /**
      * Build additional metadata for a delayed message index bucket.
      *
-     * @param bucketKey key of the delayed message bucket
+     * @param bucketKey  key of the delayed message bucket
+     * @param topicName  name of the topic
+     * @param cursorName name of the cursor
      * @return an immutable map which describes the schema
      */
-    public static Map buildMetadataForDelayedIndexBucket(String bucketKey) {
+    public static Map buildMetadataForDelayedIndexBucket(String bucketKey,
+                                                                         String topicName, String cursorName) {
         return Map.of(
                 METADATA_PROPERTY_APPLICATION, METADATA_PROPERTY_APPLICATION_PULSAR,
                 METADATA_PROPERTY_COMPONENT, METADATA_PROPERTY_COMPONENT_DELAYED_INDEX_BUCKET,
-                METADATA_PROPERTY_DELAYED_INDEX_BUCKETID, bucketKey.getBytes(StandardCharsets.UTF_8)
+                METADATA_PROPERTY_DELAYED_INDEX_BUCKET_KEY, bucketKey.getBytes(StandardCharsets.UTF_8),
+                METADATA_PROPERTY_DELAYED_INDEX_TOPIC, topicName.getBytes(StandardCharsets.UTF_8),
+                METADATA_PROPERTY_DELAYED_INDEX_CURSOR, cursorName.getBytes(StandardCharsets.UTF_8)
         );
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 7dd6266e2115c..08202bb19155d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -56,8 +56,8 @@ public BookkeeperBucketSnapshotStorage(PulsarService pulsar) {
     @Override
     public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata,
                                                         List bucketSnapshotSegments,
-                                                        String bucketKey) {
-        return createLedger(bucketKey)
+                                                        String bucketKey, String topicName, String cursorName) {
+        return createLedger(bucketKey, topicName, cursorName)
                 .thenCompose(ledgerHandle -> addEntry(ledgerHandle, snapshotMetadata.toByteArray())
                         .thenCompose(__ -> addSnapshotSegments(ledgerHandle, bucketSnapshotSegments))
                         .thenCompose(__ -> closeLedger(ledgerHandle))
@@ -143,9 +143,10 @@ private List parseSnapshotSegmentEntries(Enumeration createLedger(String bucketKey) {
+    private CompletableFuture createLedger(String bucketKey, String topicName, String cursorName) {
         CompletableFuture future = new CompletableFuture<>();
-        Map metadata = LedgerMetadataUtils.buildMetadataForDelayedIndexBucket(bucketKey);
+        Map metadata = LedgerMetadataUtils.buildMetadataForDelayedIndexBucket(bucketKey,
+                topicName, cursorName);
         bookKeeper.asyncCreateLedger(
                 config.getManagedLedgerDefaultEnsembleSize(),
                 config.getManagedLedgerDefaultWriteQuorum(),
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index db864a4e264f5..7cfccff7ba328 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -32,6 +32,7 @@
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.common.util.Codec;
 import org.roaringbitmap.RoaringBitmap;
 
 @Slf4j
@@ -130,8 +131,11 @@ CompletableFuture asyncSaveBucketSnapshot(
             ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata,
             List bucketSnapshotSegments) {
         final String bucketKey = bucket.bucketKey();
+        final String cursorName = Codec.decode(cursor.getName());
+        final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName));
         return executeWithRetry(
-                () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey)
+                () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey,
+                                topicName, cursorName)
                         .whenComplete((__, ex) -> {
                             if (ex != null) {
                                 log.warn("[{}] Failed to create bucket snapshot, bucketKey: {}",
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
index c6941e289f1ac..51c89bed47af2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
@@ -28,14 +28,16 @@ public interface BucketSnapshotStorage {
     /**
      * Create a delayed message index bucket snapshot with metadata and bucketSnapshotSegments.
      *
-     * @param snapshotMetadata the metadata of snapshot
+     * @param snapshotMetadata       the metadata of snapshot
      * @param bucketSnapshotSegments the list of snapshot segments
-     * @param bucketKey the key of bucket is used to generate custom storage metadata
+     * @param bucketKey              the key of bucket is used to generate custom storage metadata
+     * @param topicName              the name of topic is used to generate custom storage metadata
+     * @param cursorName             the name of cursor is used to generate custom storage metadata
      * @return the future with bucketId(ledgerId).
      */
     CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata,
                                                  List bucketSnapshotSegments,
-                                                 String bucketKey);
+                                                 String bucketKey, String topicName, String cursorName);
 
     /**
      * Get delayed message index bucket snapshot metadata.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 15d48e59849b6..a6345f4a56a71 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -261,6 +261,8 @@ public class BrokerService implements Closeable {
     private final ConcurrentOpenHashSet blockedDispatchers;
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
+    @Getter
+    @VisibleForTesting
     private final DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory;
     private final ServerBootstrap defaultServerBootstrap;
     private final List protocolHandlersWorkerGroups = new ArrayList<>();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
index 1effe756ff2ab..72052d22b859d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
@@ -54,6 +54,9 @@ protected void cleanup() throws Exception {
         bucketSnapshotStorage.close();
     }
 
+    private static final String TOPIC_NAME = "topicName";
+    private static final String CURSOR_NAME = "sub";
+
     @Test
     public void testCreateSnapshot() throws ExecutionException, InterruptedException {
         DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
@@ -61,7 +64,7 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException
         List bucketSnapshotSegments = new ArrayList<>();
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
     }
@@ -90,7 +93,7 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException {
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
@@ -129,7 +132,7 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
@@ -151,7 +154,7 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException
         List bucketSnapshotSegments = new ArrayList<>();
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
@@ -189,7 +192,7 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
index b106642915587..cf7310c7067c3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
@@ -80,7 +80,8 @@ public void injectDeleteException(Throwable throwable) {
 
     @Override
     public CompletableFuture createBucketSnapshot(
-            SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey) {
+            SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey,
+            String topicName, String cursorName) {
         Throwable throwable = createExceptionQueue.poll();
         if (throwable != null) {
             return FutureUtil.failedFuture(throwable);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index c834bc7a0b4c5..74101f00b960c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -79,7 +79,7 @@ public Object[][] provider(Method method) throws Exception {
         bucketSnapshotStorage.start();
         ManagedCursor cursor = new MockManagedCursor("my_test_cursor");
         doReturn(cursor).when(dispatcher).getCursor();
-        doReturn(cursor.getName()).when(dispatcher).getName();
+        doReturn("persistent://public/default/testDelay" + " / " + cursor.getName()).when(dispatcher).getName();
 
         final String methodName = method.getName();
         return switch (methodName) {

From 11cfaea0deefc3c5d32f6504d266f84d12f89641 Mon Sep 17 00:00:00 2001
From: maheshnikam <55378196+nikam14@users.noreply.github.com>
Date: Wed, 15 Mar 2023 13:01:48 +0530
Subject: [PATCH 075/404] [improve][doc] enhancement of link (#19819)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 15419a754cfa9..fdbf7c7339b1e 100644
--- a/README.md
+++ b/README.md
@@ -254,7 +254,7 @@ Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/L
 
 ## Crypto Notice
 
-This distribution includes cryptographic software. The country in which you currently reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and re-export of encryption software, to see if this is permitted. See  for more information.
+This distribution includes cryptographic software. The country in which you currently reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/) for more information.
 
 The U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has classified this software as Export Commodity Control Number (ECCN) 5D002.C.1, which includes information security software using or performing cryptographic functions with asymmetric algorithms. The form and manner of this Apache Software Foundation distribution makes it eligible for export under the License Exception ENC Technology Software Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Section 740.13) for both object code and source code.
 

From 4e48cc30bfa6bf4d01659aac34d510f8b1492a10 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Wed, 15 Mar 2023 15:57:07 +0800
Subject: [PATCH 076/404] [fix] [test] Fix flaky test
 `MetadataStoreStatsTest.testBatchMetadataStoreMetrics` (#19793)

Co-authored-by: wangjinlong 
---
 .../MultiBrokerMetadataConsistencyTest.java   | 33 ++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java
index 9eeef94572ee8..c998a1ba57c35 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.zookeeper;
 
 import static org.testng.Assert.assertTrue;
+import java.util.ArrayList;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.MultiBrokerBaseTest;
@@ -43,6 +44,9 @@ protected int numberOfAdditionalBrokers() {
 
     TestZKServer testZKServer;
 
+    private final List needCloseStore =
+            new ArrayList<>();
+
     @Override
     protected void doInitConf() throws Exception {
         super.doInitConf();
@@ -59,20 +63,41 @@ protected void onCleanup() {
                 log.error("Error in stopping ZK server", e);
             }
         }
+
+        needCloseStore.forEach((storeExtended) -> {
+            try {
+                storeExtended.close();
+            } catch (Exception e) {
+                log.error("error when close storeExtended", e);
+            }
+        });
+
+        needCloseStore.clear();
     }
 
     @Override
     protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfiguration conf) {
+        MetadataStoreExtended metadataStore = createMetadataStore(
+                MultiBrokerMetadataConsistencyTest.class.getName()
+                        + "metadata_store");
+
+        MetadataStoreExtended configurationStore = createMetadataStore(
+                MultiBrokerMetadataConsistencyTest.class.getName()
+                        + "configuration_store");
+
+        needCloseStore.add(metadataStore);
+        needCloseStore.add(configurationStore);
+
         return super.createPulsarTestContextBuilder(conf)
-                .localMetadataStore(createMetadataStore())
-                .configurationMetadataStore(createMetadataStore());
+                .localMetadataStore(metadataStore)
+                .configurationMetadataStore(configurationStore);
     }
 
     @NotNull
-    protected MetadataStoreExtended createMetadataStore()  {
+    protected MetadataStoreExtended createMetadataStore(String name) {
         try {
             return MetadataStoreExtended.create(testZKServer.getConnectionString(),
-                    MetadataStoreConfig.builder().build());
+                    MetadataStoreConfig.builder().metadataStoreName(name).build());
         } catch (MetadataStoreException e) {
             throw new RuntimeException(e);
         }

From 8cff1238465ca8b1315d8b5b48388d4b61ec74b6 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Wed, 15 Mar 2023 17:22:12 -0700
Subject: [PATCH 077/404] [improve][broker] PIP-192 Made only the leader
 consume TopBundlesLoadDataStore (#19730)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

Based on the current design, only the leader needs to consume the topk bundle load data.

### Modifications
This PR
- made only the leader consume TopBundlesLoadDataStore
- moved LeaderElectionService to ExtensibleLoadManager from ServiceUnitStateChannel.
---
 .../extensions/ExtensibleLoadManagerImpl.java | 113 +++++++++++-
 .../channel/ServiceUnitStateChannel.java      |   9 +
 .../channel/ServiceUnitStateChannelImpl.java  |  79 ++++----
 .../extensions/scheduler/UnloadScheduler.java |  11 +-
 .../extensions/store/LoadDataStore.java       |  12 ++
 .../store/TableViewLoadDataStoreImpl.java     |  46 ++++-
 .../ExtensibleLoadManagerImplTest.java        | 169 ++++++++++++++++--
 .../channel/ServiceUnitStateChannelTest.java  |  15 ++
 .../filter/BrokerFilterTestBase.java          |  11 ++
 .../scheduler/TransferShedderTest.java        |  21 +++
 .../extensions/store/LoadDataStoreTest.java   |  27 +++
 .../LeastResourceUsageWithWeightTest.java     |  12 ++
 12 files changed, 465 insertions(+), 60 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 7895f06fd5088..716be3718bf19 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -18,6 +18,9 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower;
+import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader;
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -26,6 +29,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
@@ -35,6 +39,7 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.LoadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
@@ -70,6 +75,7 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
+import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 
 @Slf4j
 public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
@@ -84,6 +90,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
             NamespaceName.SYSTEM_NAMESPACE,
             "loadbalancer-top-bundles-load-data").toString();
 
+    private static final long MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS = 200;
+
     private PulsarService pulsar;
 
     private ServiceConfiguration conf;
@@ -102,6 +110,9 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private LoadManagerScheduler unloadScheduler;
 
+    @Getter
+    private LeaderElectionService leaderElectionService;
+
     @Getter
     private LoadManagerContext context;
 
@@ -142,7 +153,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
             lookupRequests = ConcurrentOpenHashMap.>>newBuilder()
             .build();
+    private final CountDownLatch loadStoreInitWaiter = new CountDownLatch(1);
+
+    public enum Role {
+        Leader,
+        Follower
+    }
 
+    private Role role;
 
     /**
      * Life cycle: Constructor -> initialize -> start -> close.
@@ -173,12 +191,24 @@ public void start() throws PulsarServerException {
             return;
         }
         this.brokerRegistry = new BrokerRegistryImpl(pulsar);
+        this.leaderElectionService = new LeaderElectionService(
+                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
+                state -> {
+                    pulsar.getLoadManagerExecutor().execute(() -> {
+                        if (state == LeaderElectionState.Leading) {
+                            playLeader();
+                        } else {
+                            playFollower();
+                        }
+                    });
+                });
         this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
         this.brokerRegistry.start();
         this.unloadManager = new UnloadManager();
         this.splitManager = new SplitManager(splitCounter);
         this.serviceUnitStateChannel.listen(unloadManager);
         this.serviceUnitStateChannel.listen(splitManager);
+        this.leaderElectionService.start();
         this.serviceUnitStateChannel.start();
         this.antiAffinityGroupPolicyHelper =
                 new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel);
@@ -189,8 +219,10 @@ public void start() throws PulsarServerException {
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class);
+            this.brokerLoadDataStore.startTableView();
             this.topBundlesLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class);
+            this.loadStoreInitWaiter.countDown();
         } catch (LoadDataStoreException e) {
             throw new PulsarServerException(e);
         }
@@ -409,8 +441,15 @@ public void close() throws PulsarServerException {
                     this.serviceUnitStateChannel.close();
                 } finally {
                     this.unloadManager.close();
-                    this.started = false;
+                    try {
+                        this.leaderElectionService.close();
+                    } catch (Exception e) {
+                        throw new PulsarServerException(e);
+                    } finally {
+                        this.started = false;
+                    }
                 }
+
             }
         }
     }
@@ -421,6 +460,78 @@ private boolean isInternalTopic(String topic) {
                 || topic.startsWith(TOP_BUNDLES_LOAD_DATA_STORE_TOPIC);
     }
 
+    @VisibleForTesting
+    void playLeader() {
+        if (role != Leader) {
+            log.info("This broker:{} is changing the role from {} to {}",
+                    pulsar.getLookupServiceAddress(), role, Leader);
+            int retry = 0;
+            while (true) {
+                try {
+                    serviceUnitStateChannel.scheduleOwnershipMonitor();
+                    loadStoreInitWaiter.await();
+                    topBundlesLoadDataStore.startTableView();
+                    unloadScheduler.start();
+                    break;
+                } catch (Throwable e) {
+                    log.error("The broker:{} failed to change the role. Retrying {} th ...",
+                            pulsar.getLookupServiceAddress(), ++retry, e);
+                    try {
+                        Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS));
+                    } catch (InterruptedException ex) {
+                        log.warn("Interrupted while sleeping.");
+                    }
+                }
+            }
+            role = Leader;
+            log.info("This broker:{} plays the leader now.", pulsar.getLookupServiceAddress());
+        }
+
+        // flush the load data when the leader is elected.
+        if (brokerLoadDataReporter != null) {
+            brokerLoadDataReporter.reportAsync(true);
+        }
+        if (topBundleLoadDataReporter != null) {
+            topBundleLoadDataReporter.reportAsync(true);
+        }
+    }
+
+    @VisibleForTesting
+    void playFollower() {
+        if (role != Follower) {
+            log.info("This broker:{} is changing the role from {} to {}",
+                    pulsar.getLookupServiceAddress(), role, Follower);
+            int retry = 0;
+            while (true) {
+                try {
+                    serviceUnitStateChannel.cancelOwnershipMonitor();
+                    loadStoreInitWaiter.await();
+                    topBundlesLoadDataStore.closeTableView();
+                    unloadScheduler.close();
+                    break;
+                } catch (Throwable e) {
+                    log.error("The broker:{} failed to change the role. Retrying {} th ...",
+                            pulsar.getLookupServiceAddress(), ++retry, e);
+                    try {
+                        Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS));
+                    } catch (InterruptedException ex) {
+                        log.warn("Interrupted while sleeping.");
+                    }
+                }
+            }
+            role = Follower;
+            log.info("This broker:{} plays a follower now.", pulsar.getLookupServiceAddress());
+        }
+
+        // flush the load data when the leader is elected.
+        if (brokerLoadDataReporter != null) {
+            brokerLoadDataReporter.reportAsync(true);
+        }
+        if (topBundleLoadDataReporter != null) {
+            topBundleLoadDataReporter.reportAsync(true);
+        }
+    }
+
     void updateBrokerLoadMetrics(BrokerLoadData loadData) {
         this.brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress()));
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index 4e92ad791ab63..719c72a67b4ea 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -172,4 +172,13 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     Set> getOwnershipEntrySet();
 
+    /**
+     * Schedules ownership monitor to periodically check and correct invalid ownership states.
+     */
+    void scheduleOwnershipMonitor();
+
+    /**
+     * Cancels the ownership monitor.
+     */
+    void cancelOwnershipMonitor();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index f8686c07f05fa..791d02649baf1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -96,7 +96,6 @@
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.NotificationType;
-import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 import org.apache.pulsar.metadata.api.extended.SessionEvent;
 
 @Slf4j
@@ -119,9 +118,9 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     private final String lookupServiceAddress;
     private final ConcurrentOpenHashMap> cleanupJobs;
     private final StateChangeListeners stateChangeListeners;
-    private final LeaderElectionService leaderElectionService;
     private BrokerSelectionStrategy brokerSelector;
     private BrokerRegistry brokerRegistry;
+    private LeaderElectionService leaderElectionService;
     private TableView tableview;
     private Producer producer;
     private ScheduledFuture monitorTask;
@@ -205,31 +204,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) {
         }
         this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS;
         this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS;
-        this.leaderElectionService = new LeaderElectionService(
-                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
-                state -> {
-                    if (state == LeaderElectionState.Leading) {
-                        log.info("This broker:{} is the leader now.", lookupServiceAddress);
-                        this.monitorTask = this.pulsar.getLoadManagerExecutor()
-                                .scheduleWithFixedDelay(() -> {
-                                            try {
-                                                monitorOwnerships(brokerRegistry.getAvailableBrokersAsync()
-                                                        .get(inFlightStateWaitingTimeInMillis, MILLISECONDS));
-                                            } catch (Exception e) {
-                                                log.info("Failed to monitor the ownerships. will retry..", e);
-                                            }
-                                        },
-                                        ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS);
-                    } else {
-                        log.info("This broker:{} is a follower now.", lookupServiceAddress);
-                        if (monitorTask != null) {
-                            monitorTask.cancel(false);
-                            monitorTask = null;
-                            log.info("This previous leader broker:{} stopped the channel clean-up monitor",
-                                    lookupServiceAddress);
-                        }
-                    }
-                });
+
         Map tmpOwnerLookUpCounters = new HashMap<>();
         Map tmpHandlerCounters = new HashMap<>();
         Map tmpEventCounters = new HashMap<>();
@@ -246,6 +221,32 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) {
         this.channelState = Constructed;
     }
 
+    public void scheduleOwnershipMonitor() {
+        if (monitorTask == null) {
+            this.monitorTask = this.pulsar.getLoadManagerExecutor()
+                    .scheduleWithFixedDelay(() -> {
+                                try {
+                                    monitorOwnerships(brokerRegistry.getAvailableBrokersAsync()
+                                            .get(inFlightStateWaitingTimeInMillis, MILLISECONDS));
+                                } catch (Exception e) {
+                                    log.info("Failed to monitor the ownerships. will retry..", e);
+                                }
+                            },
+                            ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS);
+            log.info("This leader broker:{} started the ownership monitor.",
+                    lookupServiceAddress);
+        }
+    }
+
+    public void cancelOwnershipMonitor() {
+        if (monitorTask != null) {
+            monitorTask.cancel(false);
+            monitorTask = null;
+            log.info("This previous leader broker:{} stopped the ownership monitor.",
+                    lookupServiceAddress);
+        }
+    }
+
     public synchronized void start() throws PulsarServerException {
         if (!validateChannelState(LeaderElectionServiceStarted, false)) {
             throw new IllegalStateException("Invalid channel state:" + channelState.name());
@@ -255,11 +256,15 @@ public synchronized void start() throws PulsarServerException {
         try {
             this.brokerRegistry = getBrokerRegistry();
             this.brokerRegistry.addListener(this::handleBrokerRegistrationEvent);
-            leaderElectionService.start();
-            this.channelState = LeaderElectionServiceStarted;
-            if (debug) {
-                log.info("Successfully started the channel leader election service.");
+            this.leaderElectionService = getLeaderElectionService();
+            var leader = leaderElectionService.readCurrentLeader().get(
+                    MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS);
+            if (leader.isPresent()) {
+                log.info("Successfully found the channel leader:{}.", leader.get());
+            } else {
+                log.warn("Failed to find the channel leader.");
             }
+            this.channelState = LeaderElectionServiceStarted;
             brokerSelector = getBrokerSelector();
 
             if (producer != null) {
@@ -327,15 +332,17 @@ protected BrokerSelectionStrategy getBrokerSelector() {
         return new LeastResourceUsageWithWeight();
     }
 
+    @VisibleForTesting
+    protected LeaderElectionService getLeaderElectionService() {
+        return ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get())
+                .get().getLeaderElectionService();
+    }
+
     public synchronized void close() throws PulsarServerException {
         channelState = Closed;
         boolean debug = debug();
         try {
-            leaderElectionService.close();
-            if (debug) {
-                log.info("Successfully closed the channel leader election service.");
-            }
-
+            leaderElectionService = null;
             if (tableview != null) {
                 tableview.close();
                 tableview = null;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
index e646026978754..bc3c8eb6a94fd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
@@ -156,16 +156,19 @@ public synchronized void execute() {
 
     @Override
     public void start() {
-        long loadSheddingInterval = TimeUnit.MINUTES
-                .toMillis(conf.getLoadBalancerSheddingIntervalMinutes());
-        this.task = loadManagerExecutor.scheduleAtFixedRate(
-                this::execute, loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS);
+        if (this.task == null) {
+            long loadSheddingInterval = TimeUnit.MINUTES
+                    .toMillis(conf.getLoadBalancerSheddingIntervalMinutes());
+            this.task = loadManagerExecutor.scheduleAtFixedRate(
+                    this::execute, loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS);
+        }
     }
 
     @Override
     public void close() {
         if (this.task != null) {
             this.task.cancel(false);
+            this.task = null;
         }
         this.recentlyUnloadedBundles.clear();
         this.recentlyUnloadedBrokers.clear();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java
index 512811e10198c..680a36523a214 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.store;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -74,4 +75,15 @@ public interface LoadDataStore extends Closeable {
      */
     int size();
 
+
+    /**
+     * Closes the table view.
+     */
+    void closeTableView() throws IOException;
+
+    /**
+     * Starts the table view.
+     */
+    void startTableView() throws LoadDataStoreException;
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java
index 3909de2afa241..a400163ebf122 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java
@@ -26,6 +26,7 @@
 import java.util.function.BiConsumer;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TableView;
 
@@ -36,14 +37,22 @@
  */
 public class TableViewLoadDataStoreImpl implements LoadDataStore {
 
-    private final TableView tableView;
+    private TableView tableView;
 
     private final Producer producer;
 
+    private final PulsarClient client;
+
+    private final String topic;
+
+    private final Class clazz;
+
     public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class clazz) throws LoadDataStoreException {
         try {
-            this.tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create();
+            this.client = client;
             this.producer = client.newProducer(Schema.JSON(clazz)).topic(topic).create();
+            this.topic = topic;
+            this.clazz = clazz;
         } catch (Exception e) {
             throw new LoadDataStoreException(e);
         }
@@ -61,30 +70,59 @@ public CompletableFuture removeAsync(String key) {
 
     @Override
     public Optional get(String key) {
+        validateTableViewStart();
         return Optional.ofNullable(tableView.get(key));
     }
 
     @Override
     public void forEach(BiConsumer action) {
+        validateTableViewStart();
         tableView.forEach(action);
     }
 
     public Set> entrySet() {
+        validateTableViewStart();
         return tableView.entrySet();
     }
 
     @Override
     public int size() {
+        validateTableViewStart();
         return tableView.size();
     }
 
+    @Override
+    public void closeTableView() throws IOException {
+        if (tableView != null) {
+            tableView.close();
+            tableView = null;
+        }
+    }
+
+    @Override
+    public void startTableView() throws LoadDataStoreException {
+        if (tableView == null) {
+            try {
+                tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create();
+            } catch (PulsarClientException e) {
+                tableView = null;
+                throw new LoadDataStoreException(e);
+            }
+        }
+    }
+
     @Override
     public void close() throws IOException {
         if (producer != null) {
             producer.close();
         }
-        if (tableView != null) {
-            tableView.close();
+        closeTableView();
+    }
+
+    private void validateTableViewStart() {
+        if (tableView == null) {
+            throw new IllegalStateException("table view has not been started");
         }
     }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 17fcb9952618b..aa8583c6b57b6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -35,6 +35,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
@@ -42,6 +43,8 @@
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
@@ -49,6 +52,8 @@
 import java.util.LinkedHashMap;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -74,12 +79,14 @@
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.namespace.LookupOptions;
 import org.apache.pulsar.broker.resources.NamespaceResources;
@@ -100,6 +107,7 @@
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
@@ -143,22 +151,9 @@ public void setup() throws Exception {
         additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf);
         pulsar2 = additionalPulsarTestContext.getPulsarService();
 
-        ExtensibleLoadManagerWrapper primaryLoadManagerWrapper =
-                (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get();
-        primaryLoadManager = spy((ExtensibleLoadManagerImpl)
-                FieldUtils.readField(primaryLoadManagerWrapper, "loadManager", true));
-        FieldUtils.writeField(primaryLoadManagerWrapper, "loadManager", primaryLoadManager, true);
-
-        ExtensibleLoadManagerWrapper secondaryLoadManagerWrapper =
-                (ExtensibleLoadManagerWrapper) pulsar2.getLoadManager().get();
-        secondaryLoadManager = spy((ExtensibleLoadManagerImpl)
-                FieldUtils.readField(secondaryLoadManagerWrapper, "loadManager", true));
-        FieldUtils.writeField(secondaryLoadManagerWrapper, "loadManager", secondaryLoadManager, true);
+        setPrimaryLoadManager();
 
-        channel1 = (ServiceUnitStateChannelImpl)
-                FieldUtils.readField(primaryLoadManager, "serviceUnitStateChannel", true);
-        channel2 = (ServiceUnitStateChannelImpl)
-                FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true);
+        setSecondaryLoadManager();
 
         admin.clusters().createCluster(this.conf.getClusterName(),
                 ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
@@ -412,6 +407,130 @@ public Map filter(Map broker
         assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress());
     }
 
+    @Test
+    public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception {
+        var topBundlesLoadDataStorePrimary =
+                (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true);
+        var serviceUnitStateChannelPrimary =
+                (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager,
+                        "serviceUnitStateChannel", true);
+        var tvPrimary =
+                (TableViewImpl) FieldUtils.readDeclaredField(topBundlesLoadDataStorePrimary, "tableView", true);
+
+        var topBundlesLoadDataStoreSecondary =
+                (LoadDataStore) FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true);
+        var tvSecondary =
+                (TableViewImpl) FieldUtils.readDeclaredField(topBundlesLoadDataStoreSecondary, "tableView", true);
+
+        if (serviceUnitStateChannelPrimary.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) {
+            assertNotNull(tvPrimary);
+            assertNull(tvSecondary);
+        } else {
+            assertNull(tvPrimary);
+            assertNotNull(tvSecondary);
+        }
+
+        restartBroker();
+        pulsar1 = pulsar;
+        setPrimaryLoadManager();
+
+        var serviceUnitStateChannelPrimaryNew =
+                (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager,
+                        "serviceUnitStateChannel", true);
+        var topBundlesLoadDataStorePrimaryNew =
+                (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore"
+                        , true);
+        Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+                    assertFalse(serviceUnitStateChannelPrimaryNew.isChannelOwnerAsync().get(5, TimeUnit.SECONDS));
+                    assertNotNull(FieldUtils.readDeclaredField(topBundlesLoadDataStoreSecondary, "tableView"
+                            , true));
+                    assertNull(FieldUtils.readDeclaredField(topBundlesLoadDataStorePrimaryNew, "tableView"
+                            , true));
+                }
+        );
+    }
+
+    @Test
+    public void testRoleChange()
+            throws Exception {
+
+
+        var topBundlesLoadDataStorePrimary = (LoadDataStore)
+                FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true);
+        var topBundlesLoadDataStorePrimarySpy = spy(topBundlesLoadDataStorePrimary);
+        AtomicInteger countPri = new AtomicInteger(3);
+        AtomicInteger countPri2 = new AtomicInteger(3);
+        doAnswer(invocationOnMock -> {
+            if (countPri.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStorePrimarySpy).startTableView();
+        doAnswer(invocationOnMock -> {
+            if (countPri2.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStorePrimarySpy).closeTableView();
+        FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimarySpy, true);
+
+        var topBundlesLoadDataStoreSecondary = (LoadDataStore)
+                FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true);
+        var topBundlesLoadDataStoreSecondarySpy = spy(topBundlesLoadDataStoreSecondary);
+        AtomicInteger countSec = new AtomicInteger(3);
+        AtomicInteger countSec2 = new AtomicInteger(3);
+        doAnswer(invocationOnMock -> {
+            if (countSec.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStoreSecondarySpy).startTableView();
+        doAnswer(invocationOnMock -> {
+            if (countSec2.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStoreSecondarySpy).closeTableView();
+        FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondarySpy, true);
+
+        if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) {
+            primaryLoadManager.playFollower();
+            primaryLoadManager.playFollower();
+            secondaryLoadManager.playLeader();
+            secondaryLoadManager.playLeader();
+            primaryLoadManager.playLeader();
+            primaryLoadManager.playLeader();
+            secondaryLoadManager.playFollower();
+            secondaryLoadManager.playFollower();
+        } else {
+            primaryLoadManager.playLeader();
+            primaryLoadManager.playLeader();
+            secondaryLoadManager.playFollower();
+            secondaryLoadManager.playFollower();
+            primaryLoadManager.playFollower();
+            primaryLoadManager.playFollower();
+            secondaryLoadManager.playLeader();
+            secondaryLoadManager.playLeader();
+        }
+
+
+        verify(topBundlesLoadDataStorePrimarySpy, times(3)).startTableView();
+        verify(topBundlesLoadDataStorePrimarySpy, times(3)).closeTableView();
+        verify(topBundlesLoadDataStoreSecondarySpy, times(3)).startTableView();
+        verify(topBundlesLoadDataStoreSecondarySpy, times(3)).closeTableView();
+
+        FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimary, true);
+        FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondary, true);
+    }
+
     @Test
     public void testGetMetrics() throws Exception {
         {
@@ -612,6 +731,26 @@ private static void cleanTableView(ServiceUnitStateChannel channel)
         cache.clear();
     }
 
+    private void setPrimaryLoadManager() throws IllegalAccessException {
+        ExtensibleLoadManagerWrapper wrapper =
+                (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get();
+        primaryLoadManager = spy((ExtensibleLoadManagerImpl)
+                FieldUtils.readField(wrapper, "loadManager", true));
+        FieldUtils.writeField(wrapper, "loadManager", primaryLoadManager, true);
+        channel1 = (ServiceUnitStateChannelImpl)
+                FieldUtils.readField(primaryLoadManager, "serviceUnitStateChannel", true);
+    }
+
+    private void setSecondaryLoadManager() throws IllegalAccessException {
+        ExtensibleLoadManagerWrapper wrapper =
+                (ExtensibleLoadManagerWrapper) pulsar2.getLoadManager().get();
+        secondaryLoadManager = spy((ExtensibleLoadManagerImpl)
+                FieldUtils.readField(wrapper, "loadManager", true));
+        FieldUtils.writeField(wrapper, "loadManager", secondaryLoadManager, true);
+        channel2 = (ServiceUnitStateChannelImpl)
+                FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true);
+    }
+
     private CompletableFuture getBundleAsync(PulsarService pulsar, TopicName topic) {
         return pulsar.getNamespaceService().getBundleAsync(topic);
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 64a5f63196bea..e0bd69fce64c7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -89,6 +89,7 @@
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.NotificationType;
+import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 import org.apache.pulsar.metadata.api.extended.SessionEvent;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
@@ -1497,6 +1498,20 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar)
         doReturn(registry).when(channel).getBrokerRegistry();
         doReturn(brokerSelector).when(channel).getBrokerSelector();
 
+
+        var leaderElectionService = new LeaderElectionService(
+                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
+                state -> {
+                    if (state == LeaderElectionState.Leading) {
+                        channel.scheduleOwnershipMonitor();
+                    } else {
+                        channel.cancelOwnershipMonitor();
+                    }
+                });
+        leaderElectionService.start();
+
+        doReturn(leaderElectionService).when(channel).getLeaderElectionService();
+
         return channel;
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
index 3de957c3b1a3a..c521861471c9a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
 
 public class BrokerFilterTestBase {
@@ -82,6 +83,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
         configuration.setPreferLaterVersions(true);
         doReturn(configuration).when(mockContext).brokerConfiguration();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 47bf1ad2e7ed1..3955f1ed9af2e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -68,6 +68,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.LocalPoliciesResources;
@@ -267,6 +268,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
 
         var topBundleLoadDataStore = new LoadDataStore() {
@@ -310,6 +321,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
 
         BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java
index 5e2924cd84254..184c337a47c80 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java
@@ -20,6 +20,7 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
 import static org.testng.AssertJUnit.assertTrue;
 
 import com.google.common.collect.Sets;
@@ -74,6 +75,7 @@ public void testPushGetAndRemove() throws Exception {
         @Cleanup
         LoadDataStore loadDataStore =
                 LoadDataStoreFactory.create(pulsar.getClient(), topic, MyClass.class);
+        loadDataStore.startTableView();
         MyClass myClass1 = new MyClass("1", 1);
         loadDataStore.pushAsync("key1", myClass1).get();
 
@@ -106,6 +108,7 @@ public void testForEach() throws Exception {
         @Cleanup
         LoadDataStore loadDataStore =
                 LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class);
+        loadDataStore.startTableView();
 
         Map map = new HashMap<>();
         for (int i = 0; i < 10; i++) {
@@ -124,4 +127,28 @@ public void testForEach() throws Exception {
         assertEquals(loadDataStore.entrySet(), map.entrySet());
     }
 
+    @Test
+    public void testTableViewRestart() throws Exception {
+        String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID();
+        LoadDataStore loadDataStore =
+                LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class);
+
+        loadDataStore.startTableView();
+        loadDataStore.pushAsync("1", 1).get();
+        Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.size(), 1));
+        assertEquals(loadDataStore.get("1").get(), 1);
+        loadDataStore.closeTableView();
+
+        loadDataStore.pushAsync("1", 2).get();
+        Exception ex = null;
+        try {
+            loadDataStore.get("1");
+        } catch (IllegalStateException e) {
+            ex = e;
+        }
+        assertNotNull(ex);
+        loadDataStore.startTableView();
+        Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 2));
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java
index ebc2424cada19..0eea1d87513bf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java
@@ -24,6 +24,7 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -36,6 +37,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicName;
@@ -244,6 +246,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
 
         doReturn(conf).when(ctx).brokerConfiguration();

From 9adec1bef99fc7aedff6238f11e3d891f6e47e36 Mon Sep 17 00:00:00 2001
From: Matteo Merli 
Date: Wed, 15 Mar 2023 18:35:31 -0700
Subject: [PATCH 078/404] [fix][meta] Fixed race condition between ResourceLock
 update and invalidation (#19817)

---
 .../coordination/impl/LockManagerImpl.java    |  2 +-
 .../coordination/impl/ResourceLockImpl.java   | 56 ++++++++++++++-----
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
index 097f15af27677..ca768d38490cc 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
@@ -124,7 +124,7 @@ private void handleSessionEvent(SessionEvent se) {
             if (se == SessionEvent.SessionReestablished) {
                 log.info("Metadata store session has been re-established. Revalidating all the existing locks.");
                 for (ResourceLockImpl lock : locks.values()) {
-                    futures.add(lock.revalidate(lock.getValue(), true));
+                    futures.add(lock.revalidate(lock.getValue(), true, true));
                 }
 
             } else if (se == SessionEvent.Reconnected) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
index dea9aa1acb90f..5271a73249d80 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
@@ -44,7 +44,7 @@ public class ResourceLockImpl implements ResourceLock {
     private long version;
     private final CompletableFuture expiredFuture;
     private boolean revalidateAfterReconnection = false;
-    private CompletableFuture revalidateFuture;
+    private CompletableFuture pendingOperationFuture;
 
     private enum State {
         Init,
@@ -61,6 +61,7 @@ public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, Str
         this.path = path;
         this.version = -1;
         this.expiredFuture = new CompletableFuture<>();
+        this.pendingOperationFuture = CompletableFuture.completedFuture(null);
         this.state = State.Init;
     }
 
@@ -71,7 +72,24 @@ public synchronized T getValue() {
 
     @Override
     public synchronized CompletableFuture updateValue(T newValue) {
-       return acquire(newValue);
+        // If there is an operation in progress, we're going to let it complete before attempting to
+        // update the value
+        if (pendingOperationFuture.isDone()) {
+            pendingOperationFuture = CompletableFuture.completedFuture(null);
+        }
+
+        pendingOperationFuture = pendingOperationFuture.thenCompose(v -> {
+            synchronized (ResourceLockImpl.this) {
+                if (state != State.Valid) {
+                    return CompletableFuture.failedFuture(
+                            new IllegalStateException("Lock was not in valid state: " + state));
+                }
+
+                return acquire(newValue);
+            }
+        });
+
+        return pendingOperationFuture;
     }
 
     @Override
@@ -128,7 +146,7 @@ synchronized CompletableFuture acquire(T newValue) {
                 .thenRun(() -> result.complete(null))
                 .exceptionally(ex -> {
                     if (ex.getCause() instanceof LockBusyException) {
-                        revalidate(newValue, false)
+                        revalidate(newValue, false, false)
                                 .thenAccept(__ -> result.complete(null))
                                 .exceptionally(ex1 -> {
                                    result.completeExceptionally(ex1);
@@ -185,7 +203,7 @@ synchronized void lockWasInvalidated() {
         }
 
         log.info("Lock on resource {} was invalidated", path);
-        revalidate(value, true)
+        revalidate(value, true, true)
                 .thenRun(() -> log.info("Successfully revalidated the lock on {}", path));
     }
 
@@ -193,31 +211,39 @@ synchronized CompletableFuture revalidateIfNeededAfterReconnection() {
         if (revalidateAfterReconnection) {
             revalidateAfterReconnection = false;
             log.warn("Revalidate lock at {} after reconnection", path);
-            return revalidate(value, true);
+            return revalidate(value, true, true);
         } else {
             return CompletableFuture.completedFuture(null);
         }
     }
 
-    synchronized CompletableFuture revalidate(T newValue, boolean revalidateAfterReconnection) {
-        if (revalidateFuture == null || revalidateFuture.isDone()) {
-            revalidateFuture = doRevalidate(newValue);
+    synchronized CompletableFuture revalidate(T newValue, boolean trackPendingOperation,
+                                                    boolean revalidateAfterReconnection) {
+
+        final CompletableFuture trackFuture;
+
+        if (!trackPendingOperation) {
+            trackFuture = doRevalidate(newValue);
+        } else if (pendingOperationFuture.isDone()) {
+            pendingOperationFuture = doRevalidate(newValue);
+            trackFuture = pendingOperationFuture;
         } else {
             if (log.isDebugEnabled()) {
                 log.debug("Previous revalidating is not finished while revalidate newValue={}, value={}, version={}",
                         newValue, value, version);
             }
-            CompletableFuture newFuture = new CompletableFuture<>();
-            revalidateFuture.whenComplete((unused, throwable) -> {
-                doRevalidate(newValue).thenRun(() -> newFuture.complete(null))
+            trackFuture = new CompletableFuture<>();
+            trackFuture.whenComplete((unused, throwable) -> {
+                doRevalidate(newValue).thenRun(() -> trackFuture.complete(null))
                         .exceptionally(throwable1 -> {
-                            newFuture.completeExceptionally(throwable1);
+                            trackFuture.completeExceptionally(throwable1);
                             return null;
                         });
             });
-            revalidateFuture = newFuture;
+            pendingOperationFuture = trackFuture;
         }
-        revalidateFuture.exceptionally(ex -> {
+
+        trackFuture.exceptionally(ex -> {
             synchronized (ResourceLockImpl.this) {
                 Throwable realCause = FutureUtil.unwrapCompletionException(ex);
                 if (!revalidateAfterReconnection || realCause instanceof BadVersionException
@@ -237,7 +263,7 @@ synchronized CompletableFuture revalidate(T newValue, boolean revalidateAf
             }
             return null;
         });
-        return revalidateFuture;
+        return trackFuture;
     }
 
     private synchronized CompletableFuture doRevalidate(T newValue) {

From 0ffa97efaac32f0e2b6d951966972f22f59329b5 Mon Sep 17 00:00:00 2001
From: sinan liu 
Date: Thu, 16 Mar 2023 10:01:50 +0800
Subject: [PATCH 079/404] [cleanup][broker] Remove duplicate code in the
 SchemaRegistryServiceImpl that checks for existing schema and new schema
 types (#19753)

---
 .../schema/SchemaRegistryServiceImpl.java     | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
index 4eb87564d0f22..ae56df248d85d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
@@ -348,14 +348,14 @@ private SchemaRegistryFormat.SchemaInfo deleted(String schemaId, String user) {
 
     private void checkCompatible(SchemaAndMetadata existingSchema, SchemaData newSchema,
                                  SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException {
-        SchemaHash existingHash = SchemaHash.of(existingSchema.schema);
-        SchemaHash newHash = SchemaHash.of(newSchema);
         SchemaData existingSchemaData = existingSchema.schema;
         if (newSchema.getType() != existingSchemaData.getType()) {
             throw new IncompatibleSchemaException(String.format("Incompatible schema: "
                             + "exists schema type %s, new schema type %s",
                     existingSchemaData.getType(), newSchema.getType()));
         }
+        SchemaHash existingHash = SchemaHash.of(existingSchemaData);
+        SchemaHash newHash = SchemaHash.of(newSchema);
         if (!newHash.equals(existingHash)) {
             compatibilityChecks.getOrDefault(newSchema.getType(), SchemaCompatibilityCheck.DEFAULT)
                     .checkCompatible(existingSchemaData, newSchema, strategy);
@@ -465,17 +465,11 @@ private CompletableFuture checkCompatibilityWithLatest(String schemaId, Sc
                     }
                 });
 
-                if (existingSchema.schema.getType() != schema.getType()) {
-                    result.completeExceptionally(new IncompatibleSchemaException(
-                            String.format("Incompatible schema: exists schema type %s, new schema type %s",
-                                    existingSchema.schema.getType(), schema.getType())));
-                } else {
-                    try {
-                        checkCompatible(existingSchema, schema, strategy);
-                        result.complete(null);
-                    } catch (IncompatibleSchemaException e) {
-                        result.completeExceptionally(e);
-                    }
+                try {
+                    checkCompatible(existingSchema, schema, strategy);
+                    result.complete(null);
+                } catch (IncompatibleSchemaException e) {
+                    result.completeExceptionally(e);
                 }
                 return result;
             } else {

From da788794ebe211bc96d4d961de49ffeeb473ffad Mon Sep 17 00:00:00 2001
From: Yong Zhang 
Date: Thu, 16 Mar 2023 17:10:22 +0800
Subject: [PATCH 080/404] [improve] Upgrade bookkeeper to 4.15.4 (#19812)

### Motivation

Upgrade bookkeeper to 4.15.4.
---
 .../server/src/assemble/LICENSE.bin.txt       | 50 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        |  6 +--
 pom.xml                                       |  2 +-
 pulsar-sql/presto-distribution/LICENSE        | 24 ++++-----
 4 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 4226e58e8cb0d..6b3455127b423 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -347,31 +347,31 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.3.jar
-    - org.apache.bookkeeper-circe-checksum-4.15.3.jar
-    - org.apache.bookkeeper-cpu-affinity-4.15.3.jar
-    - org.apache.bookkeeper-statelib-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-api-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-common-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-server-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.15.3.jar
-    - org.apache.bookkeeper.http-http-server-4.15.3.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.15.3.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.3.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.3.jar
-    - org.apache.distributedlog-distributedlog-common-4.15.3.jar
-    - org.apache.distributedlog-distributedlog-core-4.15.3-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.15.3.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.15.3.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.3.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.4.jar
+    - org.apache.bookkeeper-circe-checksum-4.15.4.jar
+    - org.apache.bookkeeper-cpu-affinity-4.15.4.jar
+    - org.apache.bookkeeper-statelib-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-api-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-common-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-server-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.15.4.jar
+    - org.apache.bookkeeper.http-http-server-4.15.4.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.15.4.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.4.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.4.jar
+    - org.apache.distributedlog-distributedlog-common-4.15.4.jar
+    - org.apache.distributedlog-distributedlog-core-4.15.4-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.15.4.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.15.4.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.4.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 50608fddc9e3b..90896790b1fba 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -390,9 +390,9 @@ The Apache Software License, Version 2.0
     - log4j-web-2.18.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.15.3.jar
-    - cpu-affinity-4.15.3.jar
-    - circe-checksum-4.15.3.jar
+    - bookkeeper-common-allocator-4.15.4.jar
+    - cpu-affinity-4.15.4.jar
+    - circe-checksum-4.15.4.jar
   * AirCompressor
      - aircompressor-0.20.jar
  * AsyncHttpClient
diff --git a/pom.xml b/pom.xml
index 5b001f2d3f610..dc27ed54274fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API.
     
     1.21
 
-    4.15.3
+    4.15.4
     3.8.1
     1.5.0
     1.10.0
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 64c488c38bbe9..c523fae7606e1 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -424,18 +424,18 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
   * Apache Bookkeeper
-    - bookkeeper-common-4.15.3.jar
-    - bookkeeper-common-allocator-4.15.3.jar
-    - bookkeeper-proto-4.15.3.jar
-    - bookkeeper-server-4.15.3.jar
-    - bookkeeper-stats-api-4.15.3.jar
-    - bookkeeper-tools-framework-4.15.3.jar
-    - circe-checksum-4.15.3.jar
-    - codahale-metrics-provider-4.15.3.jar
-    - cpu-affinity-4.15.3.jar
-    - http-server-4.15.3.jar
-    - prometheus-metrics-provider-4.15.3.jar
-    - codahale-metrics-provider-4.15.3.jar
+    - bookkeeper-common-4.15.4.jar
+    - bookkeeper-common-allocator-4.15.4.jar
+    - bookkeeper-proto-4.15.4.jar
+    - bookkeeper-server-4.15.4.jar
+    - bookkeeper-stats-api-4.15.4.jar
+    - bookkeeper-tools-framework-4.15.4.jar
+    - circe-checksum-4.15.4.jar
+    - codahale-metrics-provider-4.15.4.jar
+    - cpu-affinity-4.15.4.jar
+    - http-server-4.15.4.jar
+    - prometheus-metrics-provider-4.15.4.jar
+    - codahale-metrics-provider-4.15.4.jar
   * Apache Commons
     - commons-cli-1.5.0.jar
     - commons-codec-1.15.jar

From 80c5791b87482bee3392308ecef45f455f8de885 Mon Sep 17 00:00:00 2001
From: yangyijun <1012293987@qq.com>
Date: Fri, 17 Mar 2023 01:34:54 +0800
Subject: [PATCH 081/404] [improve][broker]PIP-214 Add broker level metrics
 statistics and expose to prometheus (#19047)

---
 .../prometheus/AggregatedBrokerStats.java     |  67 +++++++++++
 .../prometheus/NamespaceStatsAggregator.java  |  48 +++++---
 .../broker/stats/PrometheusMetricsTest.java   | 105 ++++++++++++++++--
 .../broker/stats/TransactionMetricsTest.java  |   4 +-
 4 files changed, 197 insertions(+), 27 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
new file mode 100644
index 0000000000000..00c6cecdbfca1
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.stats.prometheus;
+
+public class AggregatedBrokerStats {
+    public int topicsCount;
+    public int subscriptionsCount;
+    public int producersCount;
+    public int consumersCount;
+    public double rateIn;
+    public double rateOut;
+    public double throughputIn;
+    public double throughputOut;
+    public long storageSize;
+    public long storageLogicalSize;
+    public double storageWriteRate;
+    public double storageReadRate;
+    public long msgBacklog;
+
+    void updateStats(TopicStats stats) {
+        topicsCount++;
+        subscriptionsCount += stats.subscriptionsCount;
+        producersCount += stats.producersCount;
+        consumersCount += stats.consumersCount;
+        rateIn += stats.rateIn;
+        rateOut += stats.rateOut;
+        throughputIn += stats.throughputIn;
+        throughputOut += stats.throughputOut;
+        storageSize += stats.managedLedgerStats.storageSize;
+        storageLogicalSize += stats.managedLedgerStats.storageLogicalSize;
+        storageWriteRate += stats.managedLedgerStats.storageWriteRate;
+        storageReadRate += stats.managedLedgerStats.storageReadRate;
+        msgBacklog += stats.msgBacklog;
+    }
+
+    public void reset() {
+        topicsCount = 0;
+        subscriptionsCount = 0;
+        producersCount = 0;
+        consumersCount = 0;
+        rateIn = 0;
+        rateOut = 0;
+        throughputIn = 0;
+        throughputOut = 0;
+        storageSize = 0;
+        storageLogicalSize = 0;
+        storageWriteRate = 0;
+        storageReadRate = 0;
+        msgBacklog = 0;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
index d287bf89d6c7e..918aef539cff4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
@@ -45,6 +45,14 @@
 @Slf4j
 public class NamespaceStatsAggregator {
 
+    private static final FastThreadLocal localBrokerStats =
+            new FastThreadLocal<>() {
+                @Override
+                protected AggregatedBrokerStats initialValue() {
+                    return new AggregatedBrokerStats();
+                }
+            };
+
     private static final FastThreadLocal localNamespaceStats =
             new FastThreadLocal<>() {
                 @Override
@@ -64,14 +72,13 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b
                                 boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel,
                                 PrometheusMetricStreams stream) {
         String cluster = pulsar.getConfiguration().getClusterName();
+        AggregatedBrokerStats brokerStats = localBrokerStats.get();
+        brokerStats.reset();
         AggregatedNamespaceStats namespaceStats = localNamespaceStats.get();
         TopicStats topicStats = localTopicStats.get();
         Optional compactorMXBean = getCompactorMXBean(pulsar);
         LongAdder topicsCount = new LongAdder();
         Map localNamespaceTopicCount = new HashMap<>();
-
-        printDefaultBrokerStats(stream, cluster);
-
         pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> {
             namespaceStats.reset();
             topicsCount.reset();
@@ -83,6 +90,8 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b
                         compactorMXBean
                 );
 
+                brokerStats.updateStats(topicStats);
+
                 if (includeTopicMetrics) {
                     topicsCount.add(1);
                     TopicStats.printTopicStats(stream, topicStats, compactorMXBean, cluster, namespace, name,
@@ -104,6 +113,8 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b
         if (includeTopicMetrics) {
             printTopicsCountStats(stream, localNamespaceTopicCount, cluster);
         }
+
+        printBrokerStats(stream, cluster, brokerStats);
     }
 
     private static Optional getCompactorMXBean(PulsarService pulsar) {
@@ -301,22 +312,23 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include
                 });
     }
 
-    private static void printDefaultBrokerStats(PrometheusMetricStreams stream, String cluster) {
-        // Print metrics with 0 values. This is necessary to have the available brokers being
+    private static void printBrokerStats(PrometheusMetricStreams stream, String cluster,
+                                         AggregatedBrokerStats brokerStats) {
+        // Print metrics values. This is necessary to have the available brokers being
         // reported in the brokers dashboard even if they don't have any topic or traffic
-        writeMetric(stream, "pulsar_topics_count", 0, cluster);
-        writeMetric(stream, "pulsar_subscriptions_count", 0, cluster);
-        writeMetric(stream, "pulsar_producers_count", 0, cluster);
-        writeMetric(stream, "pulsar_consumers_count", 0, cluster);
-        writeMetric(stream, "pulsar_rate_in", 0, cluster);
-        writeMetric(stream, "pulsar_rate_out", 0, cluster);
-        writeMetric(stream, "pulsar_throughput_in", 0, cluster);
-        writeMetric(stream, "pulsar_throughput_out", 0, cluster);
-        writeMetric(stream, "pulsar_storage_size", 0, cluster);
-        writeMetric(stream, "pulsar_storage_logical_size", 0, cluster);
-        writeMetric(stream, "pulsar_storage_write_rate", 0, cluster);
-        writeMetric(stream, "pulsar_storage_read_rate", 0, cluster);
-        writeMetric(stream, "pulsar_msg_backlog", 0, cluster);
+        writeMetric(stream, "pulsar_broker_topics_count", brokerStats.topicsCount, cluster);
+        writeMetric(stream, "pulsar_broker_subscriptions_count", brokerStats.subscriptionsCount, cluster);
+        writeMetric(stream, "pulsar_broker_producers_count", brokerStats.producersCount, cluster);
+        writeMetric(stream, "pulsar_broker_consumers_count", brokerStats.consumersCount, cluster);
+        writeMetric(stream, "pulsar_broker_rate_in", brokerStats.rateIn, cluster);
+        writeMetric(stream, "pulsar_broker_rate_out", brokerStats.rateOut, cluster);
+        writeMetric(stream, "pulsar_broker_throughput_in", brokerStats.throughputIn, cluster);
+        writeMetric(stream, "pulsar_broker_throughput_out", brokerStats.throughputOut, cluster);
+        writeMetric(stream, "pulsar_broker_storage_size", brokerStats.storageSize, cluster);
+        writeMetric(stream, "pulsar_broker_storage_logical_size", brokerStats.storageLogicalSize, cluster);
+        writeMetric(stream, "pulsar_broker_storage_write_rate", brokerStats.storageWriteRate, cluster);
+        writeMetric(stream, "pulsar_broker_storage_read_rate", brokerStats.storageReadRate, cluster);
+        writeMetric(stream, "pulsar_broker_msg_backlog", brokerStats.msgBacklog, cluster);
     }
 
     private static void printTopicsCountStats(PrometheusMetricStreams stream, Map namespaceTopicsCount,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index 7eb6afb97cdac..13e67762ace6f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -319,11 +319,11 @@ public void testPerTopicStats() throws Exception {
         assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("pulsar_producers_count");
-        assertEquals(cm.size(), 3);
-        assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
+        assertEquals(cm.size(), 2);
+        assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
+        assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
+        assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
         assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
-        assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
-        assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("topic_load_times_count");
         assertEquals(cm.size(), 1);
@@ -367,6 +367,97 @@ public void testPerTopicStats() throws Exception {
         c2.close();
     }
 
+    @Test
+    public void testPerBrokerStats() throws Exception {
+        Producer p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
+        Producer p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();
+
+        Consumer c1 = pulsarClient.newConsumer()
+                .topic("persistent://my-property/use/my-ns/my-topic1")
+                .subscriptionName("test")
+                .subscribe();
+
+        Consumer c2 = pulsarClient.newConsumer()
+                .topic("persistent://my-property/use/my-ns/my-topic2")
+                .subscriptionName("test")
+                .subscribe();
+
+        final int messages = 10;
+
+        for (int i = 0; i < messages; i++) {
+            String message = "my-message-" + i;
+            p1.send(message.getBytes());
+            p2.send(message.getBytes());
+        }
+
+        for (int i = 0; i < messages; i++) {
+            c1.acknowledge(c1.receive());
+            c2.acknowledge(c2.receive());
+        }
+
+        ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut);
+        String metricsStr = statsOut.toString();
+        Multimap metrics = parseMetrics(metricsStr);
+
+        Collection brokerMetrics = metrics.get("pulsar_broker_topics_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_subscriptions_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_producers_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_consumers_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_rate_in");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_rate_out");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_throughput_in");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_throughput_out");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_size");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_logical_size");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_write_rate");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_read_rate");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_msg_backlog");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        p1.close();
+        p2.close();
+        c1.close();
+        c2.close();
+    }
+
     /**
      * Test that the total message and byte counts for a topic are not reset when a consumer disconnects.
      *
@@ -674,9 +765,9 @@ public void testPerNamespaceStats() throws Exception {
         assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("pulsar_producers_count");
-        assertEquals(cm.size(), 2);
-        assertNull(cm.get(1).tags.get("topic"));
-        assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
+        assertEquals(cm.size(), 1);
+        assertNull(cm.get(0).tags.get("topic"));
+        assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("pulsar_in_bytes_total");
         assertEquals(cm.size(), 1);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java
index 30aedc022534c..4d38f5fad5141 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java
@@ -293,9 +293,9 @@ public void testManagedLedgerMetrics() throws Exception {
         metricsStr = statsOut.toString();
         metrics = parseMetrics(metricsStr);
         metric = metrics.get("pulsar_storage_size");
-        assertEquals(metric.size(), 3);
+        assertEquals(metric.size(), 2);
         metric = metrics.get("pulsar_storage_logical_size");
-        assertEquals(metric.size(), 3);
+        assertEquals(metric.size(), 2);
         metric = metrics.get("pulsar_storage_backlog_size");
         assertEquals(metric.size(), 2);
     }

From ad43c63303d9a3ec8fadffa2077dfad6ce9469fc Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Sat, 18 Mar 2023 00:10:08 +0800
Subject: [PATCH 082/404] [fix][test] Move MetadataStoreStatsTest to flaky
 group to avoid blocking merge. (#19828)

---
 .../org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
index 70e58c6b079f5..f4deddf0cec9e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
@@ -39,7 +39,7 @@
 import org.testng.annotations.Test;
 
 
-@Test(groups = "broker")
+@Test(groups = "flaky")
 public class MetadataStoreStatsTest extends BrokerTestBase {
 
     @BeforeMethod(alwaysRun = true)

From e0098ee0c34643371f53ee921bdc361e163f6387 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Sat, 18 Mar 2023 00:39:38 +0800
Subject: [PATCH 083/404] [improve][fn] Support configure compression type
 (#19470)

---
 .../common/functions/ProducerConfig.java      |   2 +
 pulsar-function-go/pb/Function.pb.go          | 654 ++++++++++--------
 pulsar-function-go/pf/instance.go             |  14 +-
 .../instance/JavaInstanceRunnable.java        |   4 +-
 .../pulsar/functions/sink/PulsarSink.java     |   6 +-
 .../instance/src/main/python/Function_pb2.py  | 131 +++-
 .../src/main/python/python_instance.py        |  13 +-
 .../proto/src/main/proto/Function.proto       |   9 +
 .../functions/utils/FunctionCommon.java       |  39 ++
 .../functions/utils/FunctionConfigUtils.java  |   8 +
 .../functions/utils/SourceConfigUtils.java    |   8 +
 .../utils/FunctionConfigUtilsTest.java        |   3 +
 .../utils/SourceConfigUtilsTest.java          |   2 +
 13 files changed, 578 insertions(+), 315 deletions(-)

diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java
index f6af7a663fa0c..25ca2ad79c877 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java
@@ -23,6 +23,7 @@
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
+import org.apache.pulsar.client.api.CompressionType;
 
 /**
  * Configuration of the producer inside the function.
@@ -38,4 +39,5 @@ public class ProducerConfig {
     private Boolean useThreadLocalProducers;
     private CryptoConfig cryptoConfig;
     private String batchBuilder;
+    private CompressionType compressionType;
 }
diff --git a/pulsar-function-go/pb/Function.pb.go b/pulsar-function-go/pb/Function.pb.go
index bee1af7cd0c22..bad1afe6658da 100644
--- a/pulsar-function-go/pb/Function.pb.go
+++ b/pulsar-function-go/pb/Function.pb.go
@@ -191,6 +191,61 @@ func (SubscriptionPosition) EnumDescriptor() ([]byte, []int) {
 	return file_Function_proto_rawDescGZIP(), []int{2}
 }
 
+type CompressionType int32
+
+const (
+	CompressionType_LZ4    CompressionType = 0
+	CompressionType_NONE   CompressionType = 1
+	CompressionType_ZLIB   CompressionType = 2
+	CompressionType_ZSTD   CompressionType = 3
+	CompressionType_SNAPPY CompressionType = 4
+)
+
+// Enum value maps for CompressionType.
+var (
+	CompressionType_name = map[int32]string{
+		0: "LZ4",
+		1: "NONE",
+		2: "ZLIB",
+		3: "ZSTD",
+		4: "SNAPPY",
+	}
+	CompressionType_value = map[string]int32{
+		"LZ4":    0,
+		"NONE":   1,
+		"ZLIB":   2,
+		"ZSTD":   3,
+		"SNAPPY": 4,
+	}
+)
+
+func (x CompressionType) Enum() *CompressionType {
+	p := new(CompressionType)
+	*p = x
+	return p
+}
+
+func (x CompressionType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (CompressionType) Descriptor() protoreflect.EnumDescriptor {
+	return file_Function_proto_enumTypes[3].Descriptor()
+}
+
+func (CompressionType) Type() protoreflect.EnumType {
+	return &file_Function_proto_enumTypes[3]
+}
+
+func (x CompressionType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use CompressionType.Descriptor instead.
+func (CompressionType) EnumDescriptor() ([]byte, []int) {
+	return file_Function_proto_rawDescGZIP(), []int{3}
+}
+
 type FunctionState int32
 
 const (
@@ -221,11 +276,11 @@ func (x FunctionState) String() string {
 }
 
 func (FunctionState) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[3].Descriptor()
+	return file_Function_proto_enumTypes[4].Descriptor()
 }
 
 func (FunctionState) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[3]
+	return &file_Function_proto_enumTypes[4]
 }
 
 func (x FunctionState) Number() protoreflect.EnumNumber {
@@ -234,7 +289,7 @@ func (x FunctionState) Number() protoreflect.EnumNumber {
 
 // Deprecated: Use FunctionState.Descriptor instead.
 func (FunctionState) EnumDescriptor() ([]byte, []int) {
-	return file_Function_proto_rawDescGZIP(), []int{3}
+	return file_Function_proto_rawDescGZIP(), []int{4}
 }
 
 type FunctionDetails_Runtime int32
@@ -270,11 +325,11 @@ func (x FunctionDetails_Runtime) String() string {
 }
 
 func (FunctionDetails_Runtime) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[4].Descriptor()
+	return file_Function_proto_enumTypes[5].Descriptor()
 }
 
 func (FunctionDetails_Runtime) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[4]
+	return &file_Function_proto_enumTypes[5]
 }
 
 func (x FunctionDetails_Runtime) Number() protoreflect.EnumNumber {
@@ -322,11 +377,11 @@ func (x FunctionDetails_ComponentType) String() string {
 }
 
 func (FunctionDetails_ComponentType) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[5].Descriptor()
+	return file_Function_proto_enumTypes[6].Descriptor()
 }
 
 func (FunctionDetails_ComponentType) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[5]
+	return &file_Function_proto_enumTypes[6]
 }
 
 func (x FunctionDetails_ComponentType) Number() protoreflect.EnumNumber {
@@ -374,11 +429,11 @@ func (x CryptoSpec_FailureAction) String() string {
 }
 
 func (CryptoSpec_FailureAction) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[6].Descriptor()
+	return file_Function_proto_enumTypes[7].Descriptor()
 }
 
 func (CryptoSpec_FailureAction) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[6]
+	return &file_Function_proto_enumTypes[7]
 }
 
 func (x CryptoSpec_FailureAction) Number() protoreflect.EnumNumber {
@@ -524,6 +579,7 @@ type FunctionDetails struct {
 	Runtime              FunctionDetails_Runtime `protobuf:"varint,8,opt,name=runtime,proto3,enum=proto.FunctionDetails_Runtime" json:"runtime,omitempty"`
 	// Deprecated since, see https://github.com/apache/pulsar/issues/15560
 	//
+	// Deprecated: Do not use.
 	AutoAck              bool                          `protobuf:"varint,9,opt,name=autoAck,proto3" json:"autoAck,omitempty"`
 	Parallelism          int32                         `protobuf:"varint,10,opt,name=parallelism,proto3" json:"parallelism,omitempty"`
 	Source               *SourceSpec                   `protobuf:"bytes,11,opt,name=source,proto3" json:"source,omitempty"`
@@ -844,11 +900,12 @@ type ProducerSpec struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	MaxPendingMessages                 int32       `protobuf:"varint,1,opt,name=maxPendingMessages,proto3" json:"maxPendingMessages,omitempty"`
-	MaxPendingMessagesAcrossPartitions int32       `protobuf:"varint,2,opt,name=maxPendingMessagesAcrossPartitions,proto3" json:"maxPendingMessagesAcrossPartitions,omitempty"`
-	UseThreadLocalProducers            bool        `protobuf:"varint,3,opt,name=useThreadLocalProducers,proto3" json:"useThreadLocalProducers,omitempty"`
-	CryptoSpec                         *CryptoSpec `protobuf:"bytes,4,opt,name=cryptoSpec,proto3" json:"cryptoSpec,omitempty"`
-	BatchBuilder                       string      `protobuf:"bytes,5,opt,name=batchBuilder,proto3" json:"batchBuilder,omitempty"`
+	MaxPendingMessages                 int32           `protobuf:"varint,1,opt,name=maxPendingMessages,proto3" json:"maxPendingMessages,omitempty"`
+	MaxPendingMessagesAcrossPartitions int32           `protobuf:"varint,2,opt,name=maxPendingMessagesAcrossPartitions,proto3" json:"maxPendingMessagesAcrossPartitions,omitempty"`
+	UseThreadLocalProducers            bool            `protobuf:"varint,3,opt,name=useThreadLocalProducers,proto3" json:"useThreadLocalProducers,omitempty"`
+	CryptoSpec                         *CryptoSpec     `protobuf:"bytes,4,opt,name=cryptoSpec,proto3" json:"cryptoSpec,omitempty"`
+	BatchBuilder                       string          `protobuf:"bytes,5,opt,name=batchBuilder,proto3" json:"batchBuilder,omitempty"`
+	CompressionType                    CompressionType `protobuf:"varint,6,opt,name=compressionType,proto3,enum=proto.CompressionType" json:"compressionType,omitempty"`
 }
 
 func (x *ProducerSpec) Reset() {
@@ -918,6 +975,13 @@ func (x *ProducerSpec) GetBatchBuilder() string {
 	return ""
 }
 
+func (x *ProducerSpec) GetCompressionType() CompressionType {
+	if x != nil {
+		return x.CompressionType
+	}
+	return CompressionType_LZ4
+}
+
 type CryptoSpec struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1017,8 +1081,7 @@ type SourceSpec struct {
 	//
 	// Deprecated: Do not use.
 	TopicsToSerDeClassName map[string]string `protobuf:"bytes,4,rep,name=topicsToSerDeClassName,proto3" json:"topicsToSerDeClassName,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	//*
-	//
+	// *
 	InputSpecs map[string]*ConsumerSpec `protobuf:"bytes,10,rep,name=inputSpecs,proto3" json:"inputSpecs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	TimeoutMs  uint64                   `protobuf:"varint,6,opt,name=timeoutMs,proto3" json:"timeoutMs,omitempty"`
 	// Deprecated: Do not use.
@@ -1030,6 +1093,7 @@ type SourceSpec struct {
 	CleanupSubscription          bool                 `protobuf:"varint,11,opt,name=cleanupSubscription,proto3" json:"cleanupSubscription,omitempty"`
 	SubscriptionPosition         SubscriptionPosition `protobuf:"varint,12,opt,name=subscriptionPosition,proto3,enum=proto.SubscriptionPosition" json:"subscriptionPosition,omitempty"`
 	NegativeAckRedeliveryDelayMs uint64               `protobuf:"varint,13,opt,name=negativeAckRedeliveryDelayMs,proto3" json:"negativeAckRedeliveryDelayMs,omitempty"`
+	SkipToLatest                 bool                 `protobuf:"varint,14,opt,name=skipToLatest,proto3" json:"skipToLatest,omitempty"`
 }
 
 func (x *SourceSpec) Reset() {
@@ -1157,6 +1221,13 @@ func (x *SourceSpec) GetNegativeAckRedeliveryDelayMs() uint64 {
 	return 0
 }
 
+func (x *SourceSpec) GetSkipToLatest() bool {
+	if x != nil {
+		return x.SkipToLatest
+	}
+	return false
+}
+
 type SinkSpec struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1173,7 +1244,7 @@ type SinkSpec struct {
 	// If specified, this will refer to an archive that is
 	// already present in the server
 	Builtin string `protobuf:"bytes,6,opt,name=builtin,proto3" json:"builtin,omitempty"`
-	//*
+	// *
 	// Builtin schema type or custom schema class name
 	SchemaType                   string            `protobuf:"bytes,7,opt,name=schemaType,proto3" json:"schemaType,omitempty"`
 	ForwardSourceMessageProperty bool              `protobuf:"varint,8,opt,name=forwardSourceMessageProperty,proto3" json:"forwardSourceMessageProperty,omitempty"`
@@ -1350,12 +1421,13 @@ type FunctionMetaData struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	FunctionDetails  *FunctionDetails            `protobuf:"bytes,1,opt,name=functionDetails,proto3" json:"functionDetails,omitempty"`
-	PackageLocation  *PackageLocationMetaData    `protobuf:"bytes,2,opt,name=packageLocation,proto3" json:"packageLocation,omitempty"`
-	Version          uint64                      `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"`
-	CreateTime       uint64                      `protobuf:"varint,4,opt,name=createTime,proto3" json:"createTime,omitempty"`
-	InstanceStates   map[int32]FunctionState     `protobuf:"bytes,5,rep,name=instanceStates,proto3" json:"instanceStates,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=proto.FunctionState"`
-	FunctionAuthSpec *FunctionAuthenticationSpec `protobuf:"bytes,6,opt,name=functionAuthSpec,proto3" json:"functionAuthSpec,omitempty"`
+	FunctionDetails                  *FunctionDetails            `protobuf:"bytes,1,opt,name=functionDetails,proto3" json:"functionDetails,omitempty"`
+	PackageLocation                  *PackageLocationMetaData    `protobuf:"bytes,2,opt,name=packageLocation,proto3" json:"packageLocation,omitempty"`
+	Version                          uint64                      `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"`
+	CreateTime                       uint64                      `protobuf:"varint,4,opt,name=createTime,proto3" json:"createTime,omitempty"`
+	InstanceStates                   map[int32]FunctionState     `protobuf:"bytes,5,rep,name=instanceStates,proto3" json:"instanceStates,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=proto.FunctionState"`
+	FunctionAuthSpec                 *FunctionAuthenticationSpec `protobuf:"bytes,6,opt,name=functionAuthSpec,proto3" json:"functionAuthSpec,omitempty"`
+	TransformFunctionPackageLocation *PackageLocationMetaData    `protobuf:"bytes,7,opt,name=transformFunctionPackageLocation,proto3" json:"transformFunctionPackageLocation,omitempty"`
 }
 
 func (x *FunctionMetaData) Reset() {
@@ -1432,18 +1504,25 @@ func (x *FunctionMetaData) GetFunctionAuthSpec() *FunctionAuthenticationSpec {
 	return nil
 }
 
+func (x *FunctionMetaData) GetTransformFunctionPackageLocation() *PackageLocationMetaData {
+	if x != nil {
+		return x.TransformFunctionPackageLocation
+	}
+	return nil
+}
+
 type FunctionAuthenticationSpec struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	//*
+	// *
 	// function authentication related data that the function authentication provider
 	// needs to cache/distribute to all workers support function authentication.
 	// Depending on the function authentication provider implementation, this can be the actual auth credentials
 	// or a pointer to the auth credentials that this function should use
 	Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
-	//*
+	// *
 	// classname of the function auth provicer this data is relevant to
 	Provider string `protobuf:"bytes,2,opt,name=provider,proto3" json:"provider,omitempty"`
 }
@@ -1776,7 +1855,7 @@ var file_Function_proto_rawDesc = []byte{
 	0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
 	0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,
 	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,
-	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9f, 0x02, 0x0a, 0x0c, 0x50, 0x72, 0x6f,
+	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x02, 0x0a, 0x0c, 0x50, 0x72, 0x6f,
 	0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2e, 0x0a, 0x12, 0x6d, 0x61, 0x78,
 	0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18,
 	0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e,
@@ -1794,202 +1873,220 @@ var file_Function_proto_rawDesc = []byte{
 	0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0a, 0x63, 0x72, 0x79, 0x70,
 	0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x62, 0x61, 0x74, 0x63, 0x68, 0x42,
 	0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x61,
-	0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x22, 0xc1, 0x03, 0x0a, 0x0a, 0x43,
-	0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x18, 0x63, 0x72, 0x79,
-	0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73,
-	0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63, 0x72, 0x79,
-	0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73,
-	0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b,
-	0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52,
-	0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a, 0x19, 0x70,
-	0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f,
-	0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x19,
-	0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69,
-	0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x1b, 0x70, 0x72, 0x6f,
-	0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75,
-	0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65,
-	0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52,
-	0x1b, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46,
-	0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x61, 0x0a, 0x1b,
-	0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61,
-	0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28,
-	0x0e, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f,
-	0x53, 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70,
-	0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22,
-	0x3d, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x49,
-	0x53, 0x43, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x53, 0x55,
-	0x4d, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0a, 0x22, 0xd1,
-	0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a,
-	0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61,
-	0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x79,
-	0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x73,
-	0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75,
-	0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x10,
-	0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x69, 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44,
-	0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53,
-	0x70, 0x65, 0x63, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44,
-	0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42,
-	0x02, 0x18, 0x01, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72,
-	0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x69,
-	0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70,
-	0x65, 0x63, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, 0x6e, 0x74,
-	0x72, 0x79, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x1c,
-	0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
-	0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x28, 0x0a, 0x0d,
-	0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x07, 0x20,
-	0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50,
-	0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69,
-	0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e,
-	0x12, 0x2a, 0x0a, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
-	0x4e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x75, 0x62, 0x73,
-	0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x13,
-	0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
-	0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6c, 0x65, 0x61, 0x6e,
-	0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f,
-	0x0a, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f,
-	0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f,
-	0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63,
-	0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12,
-	0x42, 0x0a, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x6b, 0x52, 0x65,
-	0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x4d, 0x73, 0x18,
-	0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41,
-	0x63, 0x6b, 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61,
-	0x79, 0x4d, 0x73, 0x1a, 0x49, 0x0a, 0x1b, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53,
-	0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74,
-	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x52,
-	0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72,
-	0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
-	0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75,
-	0x6d, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
-	0x38, 0x01, 0x22, 0x95, 0x05, 0x0a, 0x08, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x12,
+	0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x40, 0x0a, 0x0f, 0x63, 0x6f,
+	0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70,
+	0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0f, 0x63, 0x6f, 0x6d,
+	0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc1, 0x03, 0x0a,
+	0x0a, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x18, 0x63,
+	0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c,
+	0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63,
+	0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c,
+	0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74,
+	0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65,
+	0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a,
+	0x19, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74,
+	0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
+	0x52, 0x19, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70,
+	0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x1b, 0x70,
+	0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69,
+	0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53,
+	0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x52, 0x1b, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74,
+	0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x61,
+	0x0a, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f,
+	0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70,
+	0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72,
+	0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x22, 0x3d, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07,
+	0x44, 0x49, 0x53, 0x43, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e,
+	0x53, 0x55, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0a,
+	0x22, 0xf5, 0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12,
 	0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
 	0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a,
 	0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
 	0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43,
 	0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
-	0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a,
-	0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f,
-	0x70, 0x69, 0x63, 0x12, 0x37, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53,
-	0x70, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0c,
-	0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x26, 0x0a, 0x0e,
-	0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73,
-	0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x18,
-	0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x12, 0x1e,
-	0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x42,
-	0x0a, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x18, 0x08,
-	0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72,
-	0x74, 0x79, 0x12, 0x51, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70,
-	0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x53, 0x63,
-	0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e,
-	0x74, 0x72, 0x79, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65,
-	0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65,
-	0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28,
-	0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70,
-	0x65, 0x63, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65,
-	0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x63, 0x6f, 0x6e, 0x73,
-	0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x43,
-	0x0a, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69,
-	0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
-	0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50,
-	0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, 0x17, 0x50, 0x61,
-	0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74,
-	0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65,
-	0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b,
-	0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69,
-	0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e,
-	0x61, 0x6d, 0x65, 0x22, 0xd5, 0x03, 0x0a, 0x10, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, 0x0f, 0x66, 0x75, 0x6e, 0x63,
-	0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74,
-	0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x48, 0x0a, 0x0f, 0x70, 0x61,
-	0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b,
-	0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44,
-	0x61, 0x74, 0x61, 0x52, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e,
-	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01,
-	0x28, 0x04, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x53,
-	0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73,
-	0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46,
-	0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x2e,
-	0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e,
-	0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61,
-	0x74, 0x65, 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41,
-	0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75,
-	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63,
-	0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x53, 0x70,
-	0x65, 0x63, 0x1a, 0x57, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74,
-	0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76,
-	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65,
-	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a, 0x1a, 0x46,
-	0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74,
-	0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a,
-	0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x08, 0x49, 0x6e, 0x73,
-	0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f,
-	0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e,
-	0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
-	0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x0a, 0x41, 0x73,
-	0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74,
-	0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x69, 0x6e, 0x73,
-	0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49,
-	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49,
-	0x64, 0x2a, 0x5b, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x47,
-	0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x54, 0x4c,
-	0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x41,
-	0x54, 0x4d, 0x4f, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10,
-	0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, 0x45, 0x4c, 0x59, 0x5f, 0x4f, 0x4e, 0x43, 0x45,
-	0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x3c,
-	0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
-	0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c,
-	0x0a, 0x08, 0x46, 0x41, 0x49, 0x4c, 0x4f, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a,
-	0x4b, 0x45, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x30, 0x0a, 0x14,
-	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, 0x10, 0x00,
-	0x12, 0x0c, 0x0a, 0x08, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x01, 0x2a, 0x29,
-	0x0a, 0x0d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12,
-	0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07,
-	0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x01, 0x42, 0x2d, 0x0a, 0x21, 0x6f, 0x72, 0x67,
-	0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x70, 0x75, 0x6c, 0x73, 0x61, 0x72, 0x2e, 0x66,
-	0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x08,
-	0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a,
+	0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70,
+	0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65,
+	0x52, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x69, 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65,
+	0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63,
+	0x65, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65,
+	0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53,
+	0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a,
+	0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x53, 0x70, 0x65, 0x63, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73,
+	0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x28,
+	0x0a, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18,
+	0x07, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63,
+	0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c,
+	0x74, 0x69, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74,
+	0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x75,
+	0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30,
+	0x0a, 0x13, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
+	0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6c, 0x65,
+	0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x4f, 0x0a, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
+	0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
+	0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x73, 0x75, 0x62,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f,
+	0x6e, 0x12, 0x42, 0x0a, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x6b,
+	0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x4d,
+	0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76,
+	0x65, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65,
+	0x6c, 0x61, 0x79, 0x4d, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x54, 0x6f, 0x4c,
+	0x61, 0x74, 0x65, 0x73, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69,
+	0x70, 0x54, 0x6f, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x0a, 0x1b, 0x54, 0x6f, 0x70,
+	0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e,
+	0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x3a, 0x02, 0x38, 0x01, 0x1a, 0x52, 0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65,
+	0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x05, 0x0a, 0x08, 0x53, 0x69, 0x6e,
+	0x6b, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61,
+	0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e,
+	0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a,
+	0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e,
+	0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x37, 0x0a, 0x0c, 0x70, 0x72, 0x6f,
+	0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72,
+	0x53, 0x70, 0x65, 0x63, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70,
+	0x65, 0x63, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73,
+	0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x44,
+	0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75,
+	0x69, 0x6c, 0x74, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69,
+	0x6c, 0x74, 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79,
+	0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x42, 0x0a, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70,
+	0x65, 0x72, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x66, 0x6f, 0x72, 0x77,
+	0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x51, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65,
+	0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53,
+	0x70, 0x65, 0x63, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72,
+	0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d,
+	0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x12, 0x63,
+	0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65,
+	0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65,
+	0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
+	0x52, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72,
+	0x74, 0x69, 0x65, 0x73, 0x1a, 0x43, 0x0a, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72,
+	0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x43, 0x6f, 0x6e,
+	0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
+	0x22, 0x67, 0x0a, 0x17, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x70,
+	0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a,
+	0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61,
+	0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc1, 0x04, 0x0a, 0x10, 0x46, 0x75,
+	0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40,
+	0x0a, 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c,
+	0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52,
+	0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73,
+	0x12, 0x48, 0x0a, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61,
+	0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65,
+	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72,
+	0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69,
+	0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65,
+	0x54, 0x69, 0x6d, 0x65, 0x12, 0x53, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65,
+	0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74,
+	0x61, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74,
+	0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61,
+	0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x66, 0x75, 0x6e,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x41, 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x12, 0x6a, 0x0a, 0x20, 0x74, 0x72, 0x61, 0x6e,
+	0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x63,
+	0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61,
+	0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61,
+	0x74, 0x61, 0x52, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6e,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x57, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65,
+	0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+	0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61,
+	0x74, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a,
+	0x1a, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x64,
+	0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12,
+	0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x08, 0x49,
+	0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a,
+	0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x0a,
+	0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x69, 0x6e,
+	0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x69,
+	0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65,
+	0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65,
+	0x72, 0x49, 0x64, 0x2a, 0x5b, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e,
+	0x67, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x0c, 0x41,
+	0x54, 0x4c, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a,
+	0x0b, 0x41, 0x54, 0x4d, 0x4f, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14,
+	0x0a, 0x10, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, 0x45, 0x4c, 0x59, 0x5f, 0x4f, 0x4e,
+	0x43, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03,
+	0x2a, 0x3c, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00,
+	0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x49, 0x4c, 0x4f, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0e,
+	0x0a, 0x0a, 0x4b, 0x45, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x30,
+	0x0a, 0x14, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f,
+	0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54,
+	0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x01,
+	0x2a, 0x44, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54,
+	0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x5a, 0x34, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04,
+	0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x4c, 0x49, 0x42, 0x10, 0x02,
+	0x12, 0x08, 0x0a, 0x04, 0x5a, 0x53, 0x54, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x4e,
+	0x41, 0x50, 0x50, 0x59, 0x10, 0x04, 0x2a, 0x29, 0x0a, 0x0d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49,
+	0x4e, 0x47, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10,
+	0x01, 0x42, 0x2d, 0x0a, 0x21, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e,
+	0x70, 0x75, 0x6c, 0x73, 0x61, 0x72, 0x2e, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x08, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -2004,74 +2101,77 @@ func file_Function_proto_rawDescGZIP() []byte {
 	return file_Function_proto_rawDescData
 }
 
-var file_Function_proto_enumTypes = make([]protoimpl.EnumInfo, 7)
+var file_Function_proto_enumTypes = make([]protoimpl.EnumInfo, 8)
 var file_Function_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
 var file_Function_proto_goTypes = []interface{}{
 	(ProcessingGuarantees)(0),              // 0: proto.ProcessingGuarantees
 	(SubscriptionType)(0),                  // 1: proto.SubscriptionType
 	(SubscriptionPosition)(0),              // 2: proto.SubscriptionPosition
-	(FunctionState)(0),                     // 3: proto.FunctionState
-	(FunctionDetails_Runtime)(0),           // 4: proto.FunctionDetails.Runtime
-	(FunctionDetails_ComponentType)(0),     // 5: proto.FunctionDetails.ComponentType
-	(CryptoSpec_FailureAction)(0),          // 6: proto.CryptoSpec.FailureAction
-	(*Resources)(nil),                      // 7: proto.Resources
-	(*RetryDetails)(nil),                   // 8: proto.RetryDetails
-	(*FunctionDetails)(nil),                // 9: proto.FunctionDetails
-	(*ConsumerSpec)(nil),                   // 10: proto.ConsumerSpec
-	(*ProducerSpec)(nil),                   // 11: proto.ProducerSpec
-	(*CryptoSpec)(nil),                     // 12: proto.CryptoSpec
-	(*SourceSpec)(nil),                     // 13: proto.SourceSpec
-	(*SinkSpec)(nil),                       // 14: proto.SinkSpec
-	(*PackageLocationMetaData)(nil),        // 15: proto.PackageLocationMetaData
-	(*FunctionMetaData)(nil),               // 16: proto.FunctionMetaData
-	(*FunctionAuthenticationSpec)(nil),     // 17: proto.FunctionAuthenticationSpec
-	(*Instance)(nil),                       // 18: proto.Instance
-	(*Assignment)(nil),                     // 19: proto.Assignment
-	(*ConsumerSpec_ReceiverQueueSize)(nil), // 20: proto.ConsumerSpec.ReceiverQueueSize
-	nil,                                    // 21: proto.ConsumerSpec.SchemaPropertiesEntry
-	nil,                                    // 22: proto.ConsumerSpec.ConsumerPropertiesEntry
-	nil,                                    // 23: proto.SourceSpec.TopicsToSerDeClassNameEntry
-	nil,                                    // 24: proto.SourceSpec.InputSpecsEntry
-	nil,                                    // 25: proto.SinkSpec.SchemaPropertiesEntry
-	nil,                                    // 26: proto.SinkSpec.ConsumerPropertiesEntry
-	nil,                                    // 27: proto.FunctionMetaData.InstanceStatesEntry
+	(CompressionType)(0),                   // 3: proto.CompressionType
+	(FunctionState)(0),                     // 4: proto.FunctionState
+	(FunctionDetails_Runtime)(0),           // 5: proto.FunctionDetails.Runtime
+	(FunctionDetails_ComponentType)(0),     // 6: proto.FunctionDetails.ComponentType
+	(CryptoSpec_FailureAction)(0),          // 7: proto.CryptoSpec.FailureAction
+	(*Resources)(nil),                      // 8: proto.Resources
+	(*RetryDetails)(nil),                   // 9: proto.RetryDetails
+	(*FunctionDetails)(nil),                // 10: proto.FunctionDetails
+	(*ConsumerSpec)(nil),                   // 11: proto.ConsumerSpec
+	(*ProducerSpec)(nil),                   // 12: proto.ProducerSpec
+	(*CryptoSpec)(nil),                     // 13: proto.CryptoSpec
+	(*SourceSpec)(nil),                     // 14: proto.SourceSpec
+	(*SinkSpec)(nil),                       // 15: proto.SinkSpec
+	(*PackageLocationMetaData)(nil),        // 16: proto.PackageLocationMetaData
+	(*FunctionMetaData)(nil),               // 17: proto.FunctionMetaData
+	(*FunctionAuthenticationSpec)(nil),     // 18: proto.FunctionAuthenticationSpec
+	(*Instance)(nil),                       // 19: proto.Instance
+	(*Assignment)(nil),                     // 20: proto.Assignment
+	(*ConsumerSpec_ReceiverQueueSize)(nil), // 21: proto.ConsumerSpec.ReceiverQueueSize
+	nil,                                    // 22: proto.ConsumerSpec.SchemaPropertiesEntry
+	nil,                                    // 23: proto.ConsumerSpec.ConsumerPropertiesEntry
+	nil,                                    // 24: proto.SourceSpec.TopicsToSerDeClassNameEntry
+	nil,                                    // 25: proto.SourceSpec.InputSpecsEntry
+	nil,                                    // 26: proto.SinkSpec.SchemaPropertiesEntry
+	nil,                                    // 27: proto.SinkSpec.ConsumerPropertiesEntry
+	nil,                                    // 28: proto.FunctionMetaData.InstanceStatesEntry
 }
 var file_Function_proto_depIdxs = []int32{
 	0,  // 0: proto.FunctionDetails.processingGuarantees:type_name -> proto.ProcessingGuarantees
-	4,  // 1: proto.FunctionDetails.runtime:type_name -> proto.FunctionDetails.Runtime
-	13, // 2: proto.FunctionDetails.source:type_name -> proto.SourceSpec
-	14, // 3: proto.FunctionDetails.sink:type_name -> proto.SinkSpec
-	7,  // 4: proto.FunctionDetails.resources:type_name -> proto.Resources
-	8,  // 5: proto.FunctionDetails.retryDetails:type_name -> proto.RetryDetails
-	5,  // 6: proto.FunctionDetails.componentType:type_name -> proto.FunctionDetails.ComponentType
+	5,  // 1: proto.FunctionDetails.runtime:type_name -> proto.FunctionDetails.Runtime
+	14, // 2: proto.FunctionDetails.source:type_name -> proto.SourceSpec
+	15, // 3: proto.FunctionDetails.sink:type_name -> proto.SinkSpec
+	8,  // 4: proto.FunctionDetails.resources:type_name -> proto.Resources
+	9,  // 5: proto.FunctionDetails.retryDetails:type_name -> proto.RetryDetails
+	6,  // 6: proto.FunctionDetails.componentType:type_name -> proto.FunctionDetails.ComponentType
 	2,  // 7: proto.FunctionDetails.subscriptionPosition:type_name -> proto.SubscriptionPosition
-	20, // 8: proto.ConsumerSpec.receiverQueueSize:type_name -> proto.ConsumerSpec.ReceiverQueueSize
-	21, // 9: proto.ConsumerSpec.schemaProperties:type_name -> proto.ConsumerSpec.SchemaPropertiesEntry
-	22, // 10: proto.ConsumerSpec.consumerProperties:type_name -> proto.ConsumerSpec.ConsumerPropertiesEntry
-	12, // 11: proto.ConsumerSpec.cryptoSpec:type_name -> proto.CryptoSpec
-	12, // 12: proto.ProducerSpec.cryptoSpec:type_name -> proto.CryptoSpec
-	6,  // 13: proto.CryptoSpec.producerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
-	6,  // 14: proto.CryptoSpec.consumerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
-	1,  // 15: proto.SourceSpec.subscriptionType:type_name -> proto.SubscriptionType
-	23, // 16: proto.SourceSpec.topicsToSerDeClassName:type_name -> proto.SourceSpec.TopicsToSerDeClassNameEntry
-	24, // 17: proto.SourceSpec.inputSpecs:type_name -> proto.SourceSpec.InputSpecsEntry
-	2,  // 18: proto.SourceSpec.subscriptionPosition:type_name -> proto.SubscriptionPosition
-	11, // 19: proto.SinkSpec.producerSpec:type_name -> proto.ProducerSpec
-	25, // 20: proto.SinkSpec.schemaProperties:type_name -> proto.SinkSpec.SchemaPropertiesEntry
-	26, // 21: proto.SinkSpec.consumerProperties:type_name -> proto.SinkSpec.ConsumerPropertiesEntry
-	9,  // 22: proto.FunctionMetaData.functionDetails:type_name -> proto.FunctionDetails
-	15, // 23: proto.FunctionMetaData.packageLocation:type_name -> proto.PackageLocationMetaData
-	27, // 24: proto.FunctionMetaData.instanceStates:type_name -> proto.FunctionMetaData.InstanceStatesEntry
-	17, // 25: proto.FunctionMetaData.functionAuthSpec:type_name -> proto.FunctionAuthenticationSpec
-	16, // 26: proto.Instance.functionMetaData:type_name -> proto.FunctionMetaData
-	18, // 27: proto.Assignment.instance:type_name -> proto.Instance
-	10, // 28: proto.SourceSpec.InputSpecsEntry.value:type_name -> proto.ConsumerSpec
-	3,  // 29: proto.FunctionMetaData.InstanceStatesEntry.value:type_name -> proto.FunctionState
-	30, // [30:30] is the sub-list for method output_type
-	30, // [30:30] is the sub-list for method input_type
-	30, // [30:30] is the sub-list for extension type_name
-	30, // [30:30] is the sub-list for extension extendee
-	0,  // [0:30] is the sub-list for field type_name
+	21, // 8: proto.ConsumerSpec.receiverQueueSize:type_name -> proto.ConsumerSpec.ReceiverQueueSize
+	22, // 9: proto.ConsumerSpec.schemaProperties:type_name -> proto.ConsumerSpec.SchemaPropertiesEntry
+	23, // 10: proto.ConsumerSpec.consumerProperties:type_name -> proto.ConsumerSpec.ConsumerPropertiesEntry
+	13, // 11: proto.ConsumerSpec.cryptoSpec:type_name -> proto.CryptoSpec
+	13, // 12: proto.ProducerSpec.cryptoSpec:type_name -> proto.CryptoSpec
+	3,  // 13: proto.ProducerSpec.compressionType:type_name -> proto.CompressionType
+	7,  // 14: proto.CryptoSpec.producerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
+	7,  // 15: proto.CryptoSpec.consumerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
+	1,  // 16: proto.SourceSpec.subscriptionType:type_name -> proto.SubscriptionType
+	24, // 17: proto.SourceSpec.topicsToSerDeClassName:type_name -> proto.SourceSpec.TopicsToSerDeClassNameEntry
+	25, // 18: proto.SourceSpec.inputSpecs:type_name -> proto.SourceSpec.InputSpecsEntry
+	2,  // 19: proto.SourceSpec.subscriptionPosition:type_name -> proto.SubscriptionPosition
+	12, // 20: proto.SinkSpec.producerSpec:type_name -> proto.ProducerSpec
+	26, // 21: proto.SinkSpec.schemaProperties:type_name -> proto.SinkSpec.SchemaPropertiesEntry
+	27, // 22: proto.SinkSpec.consumerProperties:type_name -> proto.SinkSpec.ConsumerPropertiesEntry
+	10, // 23: proto.FunctionMetaData.functionDetails:type_name -> proto.FunctionDetails
+	16, // 24: proto.FunctionMetaData.packageLocation:type_name -> proto.PackageLocationMetaData
+	28, // 25: proto.FunctionMetaData.instanceStates:type_name -> proto.FunctionMetaData.InstanceStatesEntry
+	18, // 26: proto.FunctionMetaData.functionAuthSpec:type_name -> proto.FunctionAuthenticationSpec
+	16, // 27: proto.FunctionMetaData.transformFunctionPackageLocation:type_name -> proto.PackageLocationMetaData
+	17, // 28: proto.Instance.functionMetaData:type_name -> proto.FunctionMetaData
+	19, // 29: proto.Assignment.instance:type_name -> proto.Instance
+	11, // 30: proto.SourceSpec.InputSpecsEntry.value:type_name -> proto.ConsumerSpec
+	4,  // 31: proto.FunctionMetaData.InstanceStatesEntry.value:type_name -> proto.FunctionState
+	32, // [32:32] is the sub-list for method output_type
+	32, // [32:32] is the sub-list for method input_type
+	32, // [32:32] is the sub-list for extension type_name
+	32, // [32:32] is the sub-list for extension extendee
+	0,  // [0:32] is the sub-list for field type_name
 }
 
 func init() { file_Function_proto_init() }
@@ -2254,7 +2354,7 @@ func file_Function_proto_init() {
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_Function_proto_rawDesc,
-			NumEnums:      7,
+			NumEnums:      8,
 			NumMessages:   21,
 			NumExtensions: 0,
 			NumServices:   0,
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 5e05c070e65f9..5d17cfe0c333a 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -225,7 +225,19 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) {
 
 	batchBuilderType := pulsar.DefaultBatchBuilder
 
+	compressionType := pulsar.LZ4
 	if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil {
+		switch gi.context.instanceConf.funcDetails.Sink.ProducerSpec.CompressionType {
+		case pb.CompressionType_NONE:
+			compressionType = pulsar.NoCompression
+		case pb.CompressionType_ZLIB:
+			compressionType = pulsar.ZLib
+		case pb.CompressionType_ZSTD:
+			compressionType = pulsar.ZSTD
+		default:
+			compressionType = pulsar.LZ4 // go doesn't support SNAPPY yet
+		}
+
 		batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder
 		if batchBuilder != "" {
 			if batchBuilder == "KEY_BASED" {
@@ -237,7 +249,7 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) {
 	producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{
 		Topic:                   topicName,
 		Properties:              properties,
-		CompressionType:         pulsar.LZ4,
+		CompressionType:         compressionType,
 		BatchingMaxPublishDelay: time.Millisecond * 10,
 		BatcherBuilderType:      batchBuilderType,
 		SendTimeout:             0,
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
index 0dbfa0945caa7..e2ad9e4c989d1 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
@@ -904,7 +904,9 @@ private void setupOutput(ContextImpl contextImpl) throws Exception {
                             .maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions())
                             .batchBuilder(conf.getBatchBuilder())
                             .useThreadLocalProducers(conf.getUseThreadLocalProducers())
-                            .cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
+                            .cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()))
+                            .compressionType(FunctionCommon.convertFromFunctionDetailsCompressionType(
+                                    conf.getCompressionType()));
                     pulsarSinkConfig.setProducerConfig(builder.build());
                 }
 
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
index 41ec4d99e71b6..8add0a78c5fff 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
@@ -108,7 +108,6 @@ public Producer createProducer(PulsarClient client, String topic, String prod
                     .blockIfQueueFull(true)
                     .enableBatching(true)
                     .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
-                    .compressionType(CompressionType.LZ4)
                     .hashingScheme(HashingScheme.Murmur3_32Hash) //
                     .messageRoutingMode(MessageRoutingMode.CustomPartition)
                     .messageRouter(FunctionResultRouter.of())
@@ -121,6 +120,11 @@ public Producer createProducer(PulsarClient client, String topic, String prod
             }
             if (pulsarSinkConfig.getProducerConfig() != null) {
                 ProducerConfig producerConfig = pulsarSinkConfig.getProducerConfig();
+                if (producerConfig.getCompressionType() != null) {
+                    builder.compressionType(producerConfig.getCompressionType());
+                } else {
+                    builder.compressionType(CompressionType.LZ4);
+                }
                 if (producerConfig.getMaxPendingMessages() != 0) {
                     builder.maxPendingMessages(producerConfig.getMaxPendingMessages());
                 }
diff --git a/pulsar-functions/instance/src/main/python/Function_pb2.py b/pulsar-functions/instance/src/main/python/Function_pb2.py
index eebfe8589d5db..118a6a1cd8967 100644
--- a/pulsar-functions/instance/src/main/python/Function_pb2.py
+++ b/pulsar-functions/instance/src/main/python/Function_pb2.py
@@ -39,7 +39,7 @@
   syntax='proto3',
   serialized_options=b'\n!org.apache.pulsar.functions.protoB\010Function',
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa6\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x13\n\x07\x61utoAck\x18\t \x01(\x08\x42\x02\x18\x01\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xf7\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0cpoolMessages\x18\x08 \x01(\x08\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb4\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xf0\x02\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*[\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\x12\n\n\x06MANUAL\x10\x03*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3'
+  serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa6\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x13\n\x07\x61utoAck\x18\t \x01(\x08\x42\x02\x18\x01\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xf7\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0cpoolMessages\x18\x08 \x01(\x08\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xe5\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\x12/\n\x0f\x63ompressionType\x18\x06 \x01(\x0e\x32\x16.proto.CompressionType\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xba\x03\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x12H\n transformFunctionPackageLocation\x18\x07 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*[\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\x12\n\n\x06MANUAL\x10\x03*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*D\n\x0f\x43ompressionType\x12\x07\n\x03LZ4\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x08\n\x04ZLIB\x10\x02\x12\x08\n\x04ZSTD\x10\x03\x12\n\n\x06SNAPPY\x10\x04*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3'
 )
 
 _PROCESSINGGUARANTEES = _descriptor.EnumDescriptor(
@@ -72,8 +72,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3711,
-  serialized_end=3802,
+  serialized_start=3834,
+  serialized_end=3925,
 )
 _sym_db.RegisterEnumDescriptor(_PROCESSINGGUARANTEES)
 
@@ -103,8 +103,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3804,
-  serialized_end=3864,
+  serialized_start=3927,
+  serialized_end=3987,
 )
 _sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONTYPE)
 
@@ -129,12 +129,53 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3866,
-  serialized_end=3914,
+  serialized_start=3989,
+  serialized_end=4037,
 )
 _sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONPOSITION)
 
 SubscriptionPosition = enum_type_wrapper.EnumTypeWrapper(_SUBSCRIPTIONPOSITION)
+_COMPRESSIONTYPE = _descriptor.EnumDescriptor(
+  name='CompressionType',
+  full_name='proto.CompressionType',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='LZ4', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='NONE', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='ZLIB', index=2, number=2,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='ZSTD', index=3, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='SNAPPY', index=4, number=4,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=4039,
+  serialized_end=4107,
+)
+_sym_db.RegisterEnumDescriptor(_COMPRESSIONTYPE)
+
+CompressionType = enum_type_wrapper.EnumTypeWrapper(_COMPRESSIONTYPE)
 _FUNCTIONSTATE = _descriptor.EnumDescriptor(
   name='FunctionState',
   full_name='proto.FunctionState',
@@ -155,8 +196,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3916,
-  serialized_end=3957,
+  serialized_start=4109,
+  serialized_end=4150,
 )
 _sym_db.RegisterEnumDescriptor(_FUNCTIONSTATE)
 
@@ -170,6 +211,11 @@
 KEY_SHARED = 2
 LATEST = 0
 EARLIEST = 1
+LZ4 = 0
+NONE = 1
+ZLIB = 2
+ZSTD = 3
+SNAPPY = 4
 RUNNING = 0
 STOPPED = 1
 
@@ -269,8 +315,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=1899,
-  serialized_end=1960,
+  serialized_start=1948,
+  serialized_end=2009,
 )
 _sym_db.RegisterEnumDescriptor(_CRYPTOSPEC_FAILUREACTION)
 
@@ -779,6 +825,13 @@
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='compressionType', full_name='proto.ProducerSpec.compressionType', index=5,
+      number=6, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -792,7 +845,7 @@
   oneofs=[
   ],
   serialized_start=1462,
-  serialized_end=1642,
+  serialized_end=1691,
 )
 
 
@@ -852,8 +905,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1645,
-  serialized_end=1960,
+  serialized_start=1694,
+  serialized_end=2009,
 )
 
 
@@ -891,8 +944,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2440,
-  serialized_end=2501,
+  serialized_start=2489,
+  serialized_end=2550,
 )
 
 _SOURCESPEC_INPUTSPECSENTRY = _descriptor.Descriptor(
@@ -929,8 +982,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2503,
-  serialized_end=2573,
+  serialized_start=2552,
+  serialized_end=2622,
 )
 
 _SOURCESPEC = _descriptor.Descriptor(
@@ -1044,8 +1097,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1963,
-  serialized_end=2573,
+  serialized_start=2012,
+  serialized_end=2622,
 )
 
 
@@ -1222,8 +1275,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2576,
-  serialized_end=3052,
+  serialized_start=2625,
+  serialized_end=3101,
 )
 
 
@@ -1261,8 +1314,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3054,
-  serialized_end=3126,
+  serialized_start=3103,
+  serialized_end=3175,
 )
 
 
@@ -1300,8 +1353,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3422,
-  serialized_end=3497,
+  serialized_start=3545,
+  serialized_end=3620,
 )
 
 _FUNCTIONMETADATA = _descriptor.Descriptor(
@@ -1354,6 +1407,13 @@
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='transformFunctionPackageLocation', full_name='proto.FunctionMetaData.transformFunctionPackageLocation', index=6,
+      number=7, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -1366,8 +1426,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3129,
-  serialized_end=3497,
+  serialized_start=3178,
+  serialized_end=3620,
 )
 
 
@@ -1405,8 +1465,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3499,
-  serialized_end=3559,
+  serialized_start=3622,
+  serialized_end=3682,
 )
 
 
@@ -1444,8 +1504,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3561,
-  serialized_end=3642,
+  serialized_start=3684,
+  serialized_end=3765,
 )
 
 
@@ -1483,8 +1543,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3644,
-  serialized_end=3709,
+  serialized_start=3767,
+  serialized_end=3832,
 )
 
 _FUNCTIONDETAILS.fields_by_name['processingGuarantees'].enum_type = _PROCESSINGGUARANTEES
@@ -1505,6 +1565,7 @@
 _CONSUMERSPEC.fields_by_name['consumerProperties'].message_type = _CONSUMERSPEC_CONSUMERPROPERTIESENTRY
 _CONSUMERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC
 _PRODUCERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC
+_PRODUCERSPEC.fields_by_name['compressionType'].enum_type = _COMPRESSIONTYPE
 _CRYPTOSPEC.fields_by_name['producerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION
 _CRYPTOSPEC.fields_by_name['consumerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION
 _CRYPTOSPEC_FAILUREACTION.containing_type = _CRYPTOSPEC
@@ -1526,6 +1587,7 @@
 _FUNCTIONMETADATA.fields_by_name['packageLocation'].message_type = _PACKAGELOCATIONMETADATA
 _FUNCTIONMETADATA.fields_by_name['instanceStates'].message_type = _FUNCTIONMETADATA_INSTANCESTATESENTRY
 _FUNCTIONMETADATA.fields_by_name['functionAuthSpec'].message_type = _FUNCTIONAUTHENTICATIONSPEC
+_FUNCTIONMETADATA.fields_by_name['transformFunctionPackageLocation'].message_type = _PACKAGELOCATIONMETADATA
 _INSTANCE.fields_by_name['functionMetaData'].message_type = _FUNCTIONMETADATA
 _ASSIGNMENT.fields_by_name['instance'].message_type = _INSTANCE
 DESCRIPTOR.message_types_by_name['Resources'] = _RESOURCES
@@ -1544,6 +1606,7 @@
 DESCRIPTOR.enum_types_by_name['ProcessingGuarantees'] = _PROCESSINGGUARANTEES
 DESCRIPTOR.enum_types_by_name['SubscriptionType'] = _SUBSCRIPTIONTYPE
 DESCRIPTOR.enum_types_by_name['SubscriptionPosition'] = _SUBSCRIPTIONPOSITION
+DESCRIPTOR.enum_types_by_name['CompressionType'] = _COMPRESSIONTYPE
 DESCRIPTOR.enum_types_by_name['FunctionState'] = _FUNCTIONSTATE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index ac723232f4839..c679288789536 100755
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -352,6 +352,17 @@ def setup_producer(self):
       if crypto_key_reader is not None:
         encryption_key = self.instance_config.function_details.sink.producerSpec.cryptoSpec.producerEncryptionKeyName[0]
 
+      compression_type = pulsar.CompressionType.LZ4
+      if self.instance_config.function_details.sink.producerSpec.compressionType is not None:
+        if self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("NONE"):
+          compression_type = pulsar.CompressionType.NONE
+        elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("ZLIB"):
+          compression_type = pulsar.CompressionType.ZLib
+        elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("ZSTD"):
+          compression_type = pulsar.CompressionType.ZSTD
+        elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("SNAPPY"):
+          compression_type = pulsar.CompressionType.SNAPPY
+
       self.producer = self.pulsar_client.create_producer(
         str(self.instance_config.function_details.sink.topic),
         schema=self.output_schema,
@@ -359,7 +370,7 @@ def setup_producer(self):
         batching_enabled=True,
         batching_type=batch_type,
         batching_max_publish_delay_ms=10,
-        compression_type=pulsar.CompressionType.LZ4,
+        compression_type=compression_type,
         # set send timeout to be infinity to prevent potential deadlock with consumer
         # that might happen when consumer is blocked due to unacked messages
         send_timeout_millis=0,
diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto
index 101d45bc59cd7..de3f03a39008c 100644
--- a/pulsar-functions/proto/src/main/proto/Function.proto
+++ b/pulsar-functions/proto/src/main/proto/Function.proto
@@ -41,6 +41,14 @@ enum SubscriptionPosition {
     EARLIEST = 1;
 }
 
+enum CompressionType {
+    LZ4 = 0;
+    NONE = 1;
+    ZLIB = 2;
+    ZSTD = 3;
+    SNAPPY = 4;
+}
+
 message Resources {
     double cpu = 1;
     int64 ram = 2;
@@ -112,6 +120,7 @@ message ProducerSpec {
     bool useThreadLocalProducers = 3;
     CryptoSpec cryptoSpec = 4;
     string batchBuilder = 5;
+    CompressionType compressionType = 6;
 }
 
 message CryptoSpec {
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index bda99a39478a3..d3ce9d93a2d36 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -44,6 +44,7 @@
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.typetools.TypeResolver;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -568,4 +569,42 @@ public static SubscriptionInitialPosition convertFromFunctionDetailsSubscription
             return SubscriptionInitialPosition.Latest;
         }
     }
+
+    public static CompressionType convertFromFunctionDetailsCompressionType(
+            org.apache.pulsar.functions.proto.Function.CompressionType compressionType) {
+        if (compressionType == null) {
+            return CompressionType.LZ4;
+        }
+        switch (compressionType) {
+            case NONE:
+                return CompressionType.NONE;
+            case ZLIB:
+                return CompressionType.ZLIB;
+            case ZSTD:
+                return CompressionType.ZSTD;
+            case SNAPPY:
+                return CompressionType.SNAPPY;
+            default:
+                return CompressionType.LZ4;
+        }
+    }
+
+    public static org.apache.pulsar.functions.proto.Function.CompressionType convertFromCompressionType(
+       CompressionType compressionType) {
+        if (compressionType == null) {
+            return org.apache.pulsar.functions.proto.Function.CompressionType.LZ4;
+        }
+        switch (compressionType) {
+            case NONE:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.NONE;
+            case ZLIB:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.ZLIB;
+            case ZSTD:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.ZSTD;
+            case SNAPPY:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.SNAPPY;
+            default:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.LZ4;
+        }
+    }
 }
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
index d02fe5f788b5a..d20d50df1bdd1 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
@@ -24,6 +24,8 @@
 import static org.apache.commons.lang3.StringUtils.isEmpty;
 import static org.apache.pulsar.common.functions.Utils.BUILTIN;
 import static org.apache.pulsar.common.util.ClassLoaderUtils.loadJar;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType;
 import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -271,6 +273,11 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu
             if (producerConf.getBatchBuilder() != null) {
                 pbldr.setBatchBuilder(producerConf.getBatchBuilder());
             }
+            if (producerConf.getCompressionType() != null) {
+                pbldr.setCompressionType(convertFromCompressionType(producerConf.getCompressionType()));
+            } else {
+                pbldr.setCompressionType(Function.CompressionType.LZ4);
+            }
             sinkSpecBuilder.setProducerSpec(pbldr.build());
         }
         functionDetailsBuilder.setSink(sinkSpecBuilder);
@@ -471,6 +478,7 @@ public static FunctionConfig convertFromDetails(FunctionDetails functionDetails)
                 producerConfig.setBatchBuilder(spec.getBatchBuilder());
             }
             producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers());
+            producerConfig.setCompressionType(convertFromFunctionDetailsCompressionType(spec.getCompressionType()));
             functionConfig.setProducerConfig(producerConfig);
         }
         if (!isEmpty(functionDetails.getLogTopic())) {
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
index 24d4259a74a29..ec0c5c444ccba 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
@@ -19,6 +19,8 @@
 package org.apache.pulsar.functions.utils;
 
 import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType;
 import static org.apache.pulsar.functions.utils.FunctionCommon.convertProcessingGuarantee;
 import static org.apache.pulsar.functions.utils.FunctionCommon.getSourceType;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -164,6 +166,11 @@ public static FunctionDetails convert(SourceConfig sourceConfig, ExtractedSource
             if (conf.getBatchBuilder() != null) {
                 pbldr.setBatchBuilder(conf.getBatchBuilder());
             }
+            if (conf.getCompressionType() != null) {
+                pbldr.setCompressionType(convertFromCompressionType(conf.getCompressionType()));
+            } else {
+                pbldr.setCompressionType(Function.CompressionType.LZ4);
+            }
             sinkSpecBuilder.setProducerSpec(pbldr.build());
         }
 
@@ -264,6 +271,7 @@ public static SourceConfig convertFromDetails(FunctionDetails functionDetails) {
                 producerConfig.setBatchBuilder(spec.getBatchBuilder());
             }
             producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers());
+            producerConfig.setCompressionType(convertFromFunctionDetailsCompressionType(spec.getCompressionType()));
             sourceConfig.setProducerConfig(producerConfig);
         }
         if (functionDetails.hasResources()) {
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
index 4ba5138f7c6b9..8b4470d8c76c7 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
@@ -20,6 +20,7 @@
 
 import com.google.gson.Gson;
 
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.util.JsonFormat;
@@ -96,6 +97,7 @@ public void testConvertBackFidelity() {
         producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
         producerConfig.setUseThreadLocalProducers(true);
         producerConfig.setBatchBuilder("DEFAULT");
+        producerConfig.setCompressionType(CompressionType.ZLIB);
         functionConfig.setProducerConfig(producerConfig);
         Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
         FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
@@ -137,6 +139,7 @@ public void testConvertWindow() {
         producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
         producerConfig.setUseThreadLocalProducers(true);
         producerConfig.setBatchBuilder("KEY_BASED");
+        producerConfig.setCompressionType(CompressionType.SNAPPY);
         functionConfig.setProducerConfig(producerConfig);
         Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
         FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java
index 63485d7993fad..49313dbf02c62 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java
@@ -22,6 +22,7 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.Accessors;
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.ProducerConfig;
 import org.apache.pulsar.common.functions.Resources;
@@ -370,6 +371,7 @@ private SourceConfig createSourceConfig() {
         producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
         producerConfig.setUseThreadLocalProducers(true);
         producerConfig.setBatchBuilder("DEFAULT");
+        producerConfig.setCompressionType(CompressionType.ZSTD);
         sourceConfig.setProducerConfig(producerConfig);
 
         sourceConfig.setConfigs(configs);

From 4ab4463fee5f773cc29104598c707ed25f827f37 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Sat, 18 Mar 2023 17:44:45 +0800
Subject: [PATCH 084/404] [improve][broker] PIP-192: Add metrics for unload
 operation (#19749)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

We need to support metrics for unload/transfer operations in Load Manager Extension.

### Modifications
In this PR:
* Change the `findBundlesForUnloading` method return type from `UnloadDecision` to `Set`.
* The `UnloadDecision` no longer contains all unload objects. Each unload object has its own reason.
* Add units test to verify the unload counter.
---
 .../extensions/ExtensibleLoadManagerImpl.java |  18 +-
 .../extensions/manager/UnloadManager.java     |  25 +-
 .../extensions/models/UnloadCounter.java      |  72 +++--
 .../extensions/models/UnloadDecision.java     |  62 +----
 .../AntiAffinityGroupPolicyHelper.java        |   3 -
 .../scheduler/NamespaceUnloadStrategy.java    |   7 +-
 .../extensions/scheduler/TransferShedder.java |  93 ++++---
 .../extensions/scheduler/UnloadScheduler.java | 129 ++++++---
 .../ExtensibleLoadManagerImplTest.java        |  38 +--
 .../extensions/manager/UnloadManagerTest.java |  51 +++-
 .../scheduler/TransferShedderTest.java        | 260 ++++++++----------
 .../scheduler/UnloadSchedulerTest.java        |  51 +++-
 12 files changed, 440 insertions(+), 369 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 716be3718bf19..486c32153589f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -20,6 +20,8 @@
 
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower;
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -204,8 +206,8 @@ public void start() throws PulsarServerException {
                 });
         this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
         this.brokerRegistry.start();
-        this.unloadManager = new UnloadManager();
         this.splitManager = new SplitManager(splitCounter);
+        this.unloadManager = new UnloadManager(unloadCounter);
         this.serviceUnitStateChannel.listen(unloadManager);
         this.serviceUnitStateChannel.listen(splitManager);
         this.leaderElectionService.start();
@@ -265,7 +267,8 @@ public void start() throws PulsarServerException {
                         interval, TimeUnit.MILLISECONDS);
 
         this.unloadScheduler = new UnloadScheduler(
-                pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel);
+                pulsar, pulsar.getLoadManagerExecutor(), unloadManager,
+                context, serviceUnitStateChannel, antiAffinityGroupPolicyHelper, unloadCounter, unloadMetrics);
         this.unloadScheduler.start();
         this.splitScheduler = new SplitScheduler(
                 pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
@@ -401,16 +404,21 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                         log.warn(msg);
                         throw new IllegalArgumentException(msg);
                     }
-                    return unloadAsync(new Unload(sourceBroker, bundle.toString(), destinationBroker),
+                    Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker);
+                    UnloadDecision unloadDecision =
+                            new UnloadDecision(unload, Success, Admin);
+                    return unloadAsync(unloadDecision,
                             conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
                 });
     }
 
-    private CompletableFuture unloadAsync(Unload unload,
+    private CompletableFuture unloadAsync(UnloadDecision unloadDecision,
                                                long timeout,
                                                TimeUnit timeoutUnit) {
+        Unload unload = unloadDecision.getUnload();
         CompletableFuture future = serviceUnitStateChannel.publishUnloadEventAsync(unload);
-        return unloadManager.waitAsync(future, unload.serviceUnit(), timeout, timeoutUnit);
+        return unloadManager.waitAsync(future, unload.serviceUnit(), unloadDecision, timeout, timeoutUnit)
+                .thenRun(() -> unloadCounter.updateUnloadBrokerCount(1));
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java
index ead6384daba8d..2dde0c4708e41 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.manager;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -25,6 +27,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 
 /**
  * Unload manager.
@@ -32,9 +36,11 @@
 @Slf4j
 public class UnloadManager implements StateChangeListener {
 
+    private final UnloadCounter counter;
     private final Map> inFlightUnloadRequest;
 
-    public UnloadManager() {
+    public UnloadManager(UnloadCounter counter) {
+        this.counter = counter;
         this.inFlightUnloadRequest = new ConcurrentHashMap<>();
     }
 
@@ -43,14 +49,8 @@ private void complete(String serviceUnit, Throwable ex) {
             if (!future.isDone()) {
                 if (ex != null) {
                     future.completeExceptionally(ex);
-                    if (log.isDebugEnabled()) {
-                        log.debug("Complete exceptionally unload bundle: {}", serviceUnit, ex);
-                    }
                 } else {
                     future.complete(null);
-                    if (log.isDebugEnabled()) {
-                        log.debug("Complete unload bundle: {}", serviceUnit);
-                    }
                 }
             }
             return null;
@@ -59,6 +59,7 @@ private void complete(String serviceUnit, Throwable ex) {
 
     public CompletableFuture waitAsync(CompletableFuture eventPubFuture,
                                              String bundle,
+                                             UnloadDecision decision,
                                              long timeout,
                                              TimeUnit timeoutUnit) {
 
@@ -74,7 +75,15 @@ public CompletableFuture waitAsync(CompletableFuture eventPubFuture,
                 }
             });
             return future;
-        }));
+        })).whenComplete((__, ex) -> {
+            if (ex != null) {
+                counter.update(Failure, Unknown);
+                log.warn("Failed to unload bundle: {}", bundle, ex);
+                return;
+            }
+            log.info("Complete unload bundle: {}", bundle);
+            counter.update(decision);
+        });
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
index e2a51b1248967..37483b58b53e1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
@@ -21,6 +21,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
@@ -30,11 +31,13 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
+import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.apache.commons.lang3.mutable.MutableLong;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.Getter;
 import org.apache.pulsar.common.stats.Metrics;
 
 /**
@@ -45,36 +48,63 @@ public class UnloadCounter {
     long unloadBrokerCount = 0;
     long unloadBundleCount = 0;
 
-    final Map> breakdownCounters;
+    @Getter
+    @VisibleForTesting
+    final Map> breakdownCounters;
 
+    @Getter
+    @VisibleForTesting
     double loadAvg;
+    @Getter
+    @VisibleForTesting
     double loadStd;
 
+    private volatile long updatedAt = 0;
+
     public UnloadCounter() {
         breakdownCounters = Map.of(
                 Success, Map.of(
-                        Overloaded, new MutableLong(),
-                        Underloaded, new MutableLong()),
+                        Overloaded, new AtomicLong(),
+                        Underloaded, new AtomicLong(),
+                        Admin, new AtomicLong()),
                 Skip, Map.of(
-                        Balanced, new MutableLong(),
-                        NoBundles, new MutableLong(),
-                        CoolDown, new MutableLong(),
-                        OutDatedData, new MutableLong(),
-                        NoLoadData, new MutableLong(),
-                        NoBrokers, new MutableLong(),
-                        Unknown, new MutableLong()),
+                        Balanced, new AtomicLong(),
+                        NoBundles, new AtomicLong(),
+                        CoolDown, new AtomicLong(),
+                        OutDatedData, new AtomicLong(),
+                        NoLoadData, new AtomicLong(),
+                        NoBrokers, new AtomicLong(),
+                        Unknown, new AtomicLong()),
                 Failure, Map.of(
-                        Unknown, new MutableLong())
+                        Unknown, new AtomicLong())
         );
     }
 
     public void update(UnloadDecision decision) {
-        var unloads = decision.getUnloads();
-        unloadBrokerCount += unloads.keySet().size();
-        unloadBundleCount += unloads.values().size();
-        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment();
-        loadAvg = decision.loadAvg;
-        loadStd = decision.loadStd;
+        if (decision.getLabel() == Success) {
+            unloadBundleCount++;
+        }
+        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void update(UnloadDecision.Label label, UnloadDecision.Reason reason) {
+        if (label == Success) {
+            unloadBundleCount++;
+        }
+        breakdownCounters.get(label).get(reason).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void updateLoadData(double loadAvg, double loadStd) {
+        this.loadAvg = loadAvg;
+        this.loadStd = loadStd;
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void updateUnloadBrokerCount(int unloadBrokerCount) {
+        this.unloadBrokerCount += unloadBrokerCount;
+        updatedAt = System.currentTimeMillis();
     }
 
     public List toMetrics(String advertisedBrokerAddress) {
@@ -125,4 +155,8 @@ public List toMetrics(String advertisedBrokerAddress) {
 
         return metrics;
     }
-}
\ No newline at end of file
+
+    public long updatedAt() {
+        return updatedAt;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
index 67503db34eee7..e1087ab6e53ba 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
@@ -18,29 +18,21 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
+import lombok.AllArgsConstructor;
 import lombok.Data;
 
 /**
  * Defines the information required to unload or transfer a service unit(e.g. bundle).
  */
 @Data
+@AllArgsConstructor
 public class UnloadDecision {
-    Multimap unloads;
+    Unload unload;
     Label label;
     Reason reason;
-    Double loadAvg;
-    Double loadStd;
+
     public enum Label {
         Success,
         Skip,
@@ -55,39 +47,20 @@ public enum Reason {
         OutDatedData,
         NoLoadData,
         NoBrokers,
+        Admin,
         Unknown
     }
 
     public UnloadDecision() {
-        unloads = ArrayListMultimap.create();
+        unload = null;
         label = null;
         reason = null;
-        loadAvg = null;
-        loadStd = null;
     }
 
     public void clear() {
-        unloads.clear();
+        unload = null;
         label = null;
         reason = null;
-        loadAvg = null;
-        loadStd = null;
-    }
-
-    public void skip(int numOfOverloadedBrokers,
-                     int numOfUnderloadedBrokers,
-                     int numOfBrokersWithEmptyLoadData,
-                     int numOfBrokersWithFewBundles) {
-        label = Skip;
-        if (numOfOverloadedBrokers == 0 && numOfUnderloadedBrokers == 0) {
-            reason = Balanced;
-        } else if (numOfBrokersWithEmptyLoadData > 0) {
-            reason = NoLoadData;
-        } else if (numOfBrokersWithFewBundles > 0) {
-            reason = NoBundles;
-        } else {
-            reason = Unknown;
-        }
     }
 
     public void skip(Reason reason) {
@@ -95,22 +68,9 @@ public void skip(Reason reason) {
         this.reason = reason;
     }
 
-    public void succeed(
-                        int numOfOverloadedBrokers,
-                        int numOfUnderloadedBrokers) {
-
-        label = Success;
-        if (numOfOverloadedBrokers > numOfUnderloadedBrokers) {
-            reason = Overloaded;
-        } else {
-            reason = Underloaded;
-        }
-    }
-
-
-    public void fail() {
-        label = Failure;
-        reason = Unknown;
+    public void succeed(Reason reason) {
+        this.label = Success;
+        this.reason = reason;
     }
 
-}
\ No newline at end of file
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
index 28acf5fba0ea1..c8332a1d7b5e6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -54,9 +54,6 @@ public boolean canUnload(
             String bundle,
             String srcBroker,
             Optional dstBroker) {
-
-
-
         try {
             var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
                     pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
index b4dc92d92187d..42af396abcad4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
 import java.util.Map;
+import java.util.Set;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 
@@ -38,8 +39,8 @@ public interface NamespaceUnloadStrategy {
      * @param recentlyUnloadedBrokers The recently unloaded brokers.
      * @return unloadDecision containing a list of the bundles that should be unloaded.
      */
-    UnloadDecision findBundlesForUnloading(LoadManagerContext context,
-                                           Map recentlyUnloadedBundles,
-                                           Map recentlyUnloadedBrokers);
+    Set findBundlesForUnloading(LoadManagerContext context,
+                                                Map recentlyUnloadedBundles,
+                                                Map recentlyUnloadedBrokers);
 
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 9f9582df2cc28..3c67479bcc7cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -18,12 +18,20 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.MinMaxPriorityQueue;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -39,6 +47,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
@@ -80,19 +89,26 @@ public class TransferShedder implements NamespaceUnloadStrategy {
     private final IsolationPoliciesHelper isolationPoliciesHelper;
     private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
-    private final UnloadDecision decision = new UnloadDecision();
+    private final Set decisionCache;
+    private final UnloadCounter counter;
 
     @VisibleForTesting
-    public TransferShedder(){
+    public TransferShedder(UnloadCounter counter){
         this.pulsar = null;
+        this.decisionCache = new HashSet<>();
         this.allocationPolicies = null;
+        this.counter = counter;
         this.isolationPoliciesHelper = null;
         this.antiAffinityGroupPolicyHelper = null;
     }
 
-    public TransferShedder(PulsarService pulsar, AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper) {
+    public TransferShedder(PulsarService pulsar,
+                           UnloadCounter counter,
+                           AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper){
         this.pulsar = pulsar;
+        this.decisionCache = new HashSet<>();
         this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
+        this.counter = counter;
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
         this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
     }
@@ -241,13 +257,12 @@ public String toString() {
 
 
     @Override
-    public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
+    public Set findBundlesForUnloading(LoadManagerContext context,
                                                   Map recentlyUnloadedBundles,
                                                   Map recentlyUnloadedBrokers) {
         final var conf = context.brokerConfiguration();
-        decision.clear();
+        decisionCache.clear();
         stats.clear();
-        var selectedBundlesCache = decision.getUnloads();
 
         try {
             final var loadStore = context.brokerLoadDataStore();
@@ -255,22 +270,17 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
             boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
 
             var skipReason = stats.update(context.brokerLoadDataStore(), recentlyUnloadedBrokers, conf);
-            if (!skipReason.isEmpty()) {
-                decision.skip(skipReason.get());
-                log.warn("Failed to update load stat. Reason:{}. Stop unloading.", decision.getReason());
-                return decision;
+            if (skipReason.isPresent()) {
+                log.warn("Failed to update load stat. Reason:{}. Stop unloading.", skipReason.get());
+                counter.update(Skip, skipReason.get());
+                return decisionCache;
             }
-            decision.setLoadAvg(stats.avg);
-            decision.setLoadStd(stats.std);
+            counter.updateLoadData(stats.avg, stats.std);
 
             if (debugMode) {
                 log.info("brokers' load stats:{}", stats);
             }
 
-            // success metrics
-            int numOfOverloadedBrokers = 0;
-            int numOfUnderloadedBrokers = 0;
-
             // skip metrics
             int numOfBrokersWithEmptyLoadData = 0;
             int numOfBrokersWithFewBundles = 0;
@@ -283,9 +293,9 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
                 availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync()
                         .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
             } catch (ExecutionException | InterruptedException | TimeoutException e) {
-                decision.skip(Unknown);
-                log.warn("Failed to fetch available brokers. Reason:{}. Stop unloading.", decision.getReason(), e);
-                return decision;
+                counter.update(Skip, Unknown);
+                log.warn("Failed to fetch available brokers. Reason: Unknown. Stop unloading.", e);
+                return decisionCache;
             }
 
             while (true) {
@@ -295,6 +305,7 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
                     }
                     break;
                 }
+                UnloadDecision.Reason reason;
                 if (stats.std() <= targetStd) {
                     if (hasMsgThroughput(context, stats.minBrokers.peekLast())) {
                         if (debugMode) {
@@ -303,10 +314,10 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
                         }
                         break;
                     } else {
-                        numOfUnderloadedBrokers++;
+                        reason = Underloaded;
                     }
                 } else {
-                    numOfOverloadedBrokers++;
+                    reason = Overloaded;
                 }
 
                 String maxBroker = stats.maxBrokers().pollLast();
@@ -365,13 +376,16 @@ && isTransferable(context, availableBrokers,
                             if (remainingTopBundles > 1
                                     && (trafficMarkedToOffload < offloadThroughput
                                     || !atLeastOneBundleSelected)) {
+                                Unload unload;
                                 if (transfer) {
-                                    selectedBundlesCache.put(maxBroker,
-                                            new Unload(maxBroker, bundle, Optional.of(minBroker)));
+                                    unload = new Unload(maxBroker, bundle, Optional.of(minBroker));
                                 } else {
-                                    selectedBundlesCache.put(maxBroker,
-                                            new Unload(maxBroker, bundle));
+                                    unload = new Unload(maxBroker, bundle);
                                 }
+                                var decision = new UnloadDecision();
+                                decision.setUnload(unload);
+                                decision.succeed(reason);
+                                decisionCache.add(decision);
                                 trafficMarkedToOffload += throughput;
                                 atLeastOneBundleSelected = true;
                                 remainingTopBundles--;
@@ -403,26 +417,24 @@ && isTransferable(context, availableBrokers,
             }
 
             if (debugMode) {
-                log.info("selectedBundlesCache:{}", selectedBundlesCache);
+                log.info("decisionCache:{}", decisionCache);
             }
-
-            if (decision.getUnloads().isEmpty()) {
-                decision.skip(
-                        numOfOverloadedBrokers,
-                        numOfUnderloadedBrokers,
-                        numOfBrokersWithEmptyLoadData,
-                        numOfBrokersWithFewBundles);
-            } else {
-                decision.succeed(
-                        numOfOverloadedBrokers,
-                        numOfUnderloadedBrokers);
+            if (decisionCache.isEmpty()) {
+                UnloadDecision.Reason reason;
+                if (numOfBrokersWithEmptyLoadData > 0) {
+                    reason = NoLoadData;
+                } else if (numOfBrokersWithFewBundles > 0) {
+                    reason = NoBundles;
+                } else {
+                    reason = Balanced;
+                }
+                counter.update(Skip, reason);
             }
         } catch (Throwable e) {
             log.error("Failed to process unloading. ", e);
-            decision.fail();
+            this.counter.update(Failure, Unknown);
         }
-
-        return decision;
+        return decisionCache;
     }
 
 
@@ -457,7 +469,6 @@ private boolean isTransferable(LoadManagerContext context,
         if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
             return false;
         }
-
         return true;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
index bc3c8eb6a94fd..31310c5c9cc80 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
@@ -18,21 +18,30 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.Reflections;
 
@@ -43,6 +52,8 @@ public class UnloadScheduler implements LoadManagerScheduler {
 
     private final ScheduledExecutorService loadManagerExecutor;
 
+    private final PulsarService pulsar;
+
     private final UnloadManager unloadManager;
 
     private final LoadManagerContext context;
@@ -51,32 +62,49 @@ public class UnloadScheduler implements LoadManagerScheduler {
 
     private final ServiceConfiguration conf;
 
+    private final UnloadCounter counter;
+
+    private final AtomicReference> unloadMetrics;
+
+    private long counterLastUpdatedAt = 0;
+
     private volatile ScheduledFuture task;
 
+    private final Set unloadBrokers;
+
     private final Map recentlyUnloadedBundles;
 
     private final Map recentlyUnloadedBrokers;
 
-    private volatile CompletableFuture currentRunningFuture = null;
-
-    public UnloadScheduler(ScheduledExecutorService loadManagerExecutor,
+    public UnloadScheduler(PulsarService pulsar,
+                           ScheduledExecutorService loadManagerExecutor,
                            UnloadManager unloadManager,
                            LoadManagerContext context,
-                           ServiceUnitStateChannel channel) {
-        this(loadManagerExecutor, unloadManager, context,
-                channel, createNamespaceUnloadStrategy(context.brokerConfiguration()));
+                           ServiceUnitStateChannel channel,
+                           AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper,
+                           UnloadCounter counter,
+                           AtomicReference> unloadMetrics) {
+        this(pulsar, loadManagerExecutor, unloadManager, context, channel,
+                createNamespaceUnloadStrategy(pulsar, antiAffinityGroupPolicyHelper, counter), counter, unloadMetrics);
     }
 
     @VisibleForTesting
-    protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor,
+    protected UnloadScheduler(PulsarService pulsar,
+                              ScheduledExecutorService loadManagerExecutor,
                               UnloadManager unloadManager,
                               LoadManagerContext context,
                               ServiceUnitStateChannel channel,
-                              NamespaceUnloadStrategy strategy) {
+                              NamespaceUnloadStrategy strategy,
+                              UnloadCounter counter,
+                              AtomicReference> unloadMetrics) {
+        this.pulsar = pulsar;
         this.namespaceUnloadStrategy = strategy;
         this.recentlyUnloadedBundles = new HashMap<>();
         this.recentlyUnloadedBrokers = new HashMap<>();
+        this.unloadBrokers = new HashSet<>();
         this.loadManagerExecutor = loadManagerExecutor;
+        this.counter = counter;
+        this.unloadMetrics = unloadMetrics;
         this.unloadManager = unloadManager;
         this.context = context;
         this.conf = context.brokerConfiguration();
@@ -96,62 +124,72 @@ public synchronized void execute() {
             }
             return;
         }
-        if (currentRunningFuture != null && !currentRunningFuture.isDone()) {
-            if (debugMode) {
-                log.info("Auto namespace unload is running. Skipping.");
-            }
-            return;
-        }
         // Remove bundles who have been unloaded for longer than the grace period from the recently unloaded map.
         final long timeout = System.currentTimeMillis()
                 - TimeUnit.MINUTES.toMillis(conf.getLoadBalancerSheddingGracePeriodMinutes());
         recentlyUnloadedBundles.keySet().removeIf(e -> recentlyUnloadedBundles.get(e) < timeout);
 
-        this.currentRunningFuture = channel.isChannelOwnerAsync().thenCompose(isChannelOwner -> {
-            if (!isChannelOwner) {
-                if (debugMode) {
-                    log.info("Current broker is not channel owner. Skipping.");
+        long asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs();
+        synchronized (namespaceUnloadStrategy) {
+            try {
+                Boolean isChannelOwner = channel.isChannelOwnerAsync().get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
+                if (!isChannelOwner) {
+                    if (debugMode) {
+                        log.info("Current broker is not channel owner. Skipping.");
+                    }
+                    return;
                 }
-                return CompletableFuture.completedFuture(null);
-            }
-            return context.brokerRegistry().getAvailableBrokersAsync().thenCompose(availableBrokers -> {
+                List availableBrokers = context.brokerRegistry().getAvailableBrokersAsync()
+                        .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
                 if (debugMode) {
-                   log.info("Available brokers: {}", availableBrokers);
+                    log.info("Available brokers: {}", availableBrokers);
                 }
                 if (availableBrokers.size() <= 1) {
                     log.info("Only 1 broker available: no load shedding will be performed. Skipping.");
-                    return CompletableFuture.completedFuture(null);
+                    return;
                 }
-                final UnloadDecision unloadDecision = namespaceUnloadStrategy
+                final Set decisions = namespaceUnloadStrategy
                         .findBundlesForUnloading(context, recentlyUnloadedBundles, recentlyUnloadedBrokers);
                 if (debugMode) {
                     log.info("[{}] Unload decision result: {}",
-                            namespaceUnloadStrategy.getClass().getSimpleName(), unloadDecision.toString());
+                            namespaceUnloadStrategy.getClass().getSimpleName(), decisions);
                 }
-                if (unloadDecision.getUnloads().isEmpty()) {
+                if (decisions.isEmpty()) {
                     if (debugMode) {
                         log.info("[{}] Unload decision unloads is empty. Skipping.",
                                 namespaceUnloadStrategy.getClass().getSimpleName());
                     }
-                    return CompletableFuture.completedFuture(null);
+                    return;
                 }
                 List> futures = new ArrayList<>();
-                unloadDecision.getUnloads().forEach((broker, unload) -> {
-                    log.info("[{}] Unloading bundle: {}", namespaceUnloadStrategy.getClass().getSimpleName(), unload);
-                    futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload), unload.serviceUnit(),
-                                    conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS)
-                            .thenAccept(__ -> {
-                                recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis());
-                                recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis());
-                    }));
-                });
-                return FutureUtil.waitForAll(futures).exceptionally(ex -> {
-                    log.error("[{}] Namespace unload has exception.",
-                            namespaceUnloadStrategy.getClass().getSimpleName(), ex);
-                    return null;
+                unloadBrokers.clear();
+                decisions.forEach(decision -> {
+                    if (decision.getLabel() == Success) {
+                        Unload unload = decision.getUnload();
+                        log.info("[{}] Unloading bundle: {}",
+                                namespaceUnloadStrategy.getClass().getSimpleName(), unload);
+                        futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload),
+                                        unload.serviceUnit(), decision, asyncOpTimeoutMs, TimeUnit.MILLISECONDS)
+                                .thenAccept(__ -> {
+                                    unloadBrokers.add(unload.sourceBroker());
+                                    recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis());
+                                    recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis());
+                                }));
+                    }
                 });
-            });
-        });
+                FutureUtil.waitForAll(futures)
+                        .whenComplete((__, ex) -> counter.updateUnloadBrokerCount(unloadBrokers.size()))
+                        .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
+            } catch (Exception ex) {
+                log.error("[{}] Namespace unload has exception.",
+                        namespaceUnloadStrategy.getClass().getSimpleName(), ex);
+            } finally {
+                if (counter.updatedAt() > counterLastUpdatedAt) {
+                    unloadMetrics.set(counter.toMetrics(pulsar.getAdvertisedAddress()));
+                    counterLastUpdatedAt = counter.updatedAt();
+                }
+            }
+        }
     }
 
     @Override
@@ -174,7 +212,10 @@ public void close() {
         this.recentlyUnloadedBrokers.clear();
     }
 
-    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConfiguration conf) {
+    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar,
+                                                                         AntiAffinityGroupPolicyHelper helper,
+                                                                         UnloadCounter counter) {
+        ServiceConfiguration conf = pulsar.getConfiguration();
         try {
             return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class,
                     Thread.currentThread().getContextClassLoader());
@@ -183,7 +224,7 @@ private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConf
                     conf.getLoadBalancerLoadPlacementStrategy(), e);
         }
         log.error("create namespace unload strategy failed. using TransferShedder instead.");
-        return new TransferShedder();
+        return new TransferShedder(pulsar, counter, helper);
     }
 
     private boolean isLoadBalancerSheddingEnabled() {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index aa8583c6b57b6..354a27a63c02a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -64,8 +64,6 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentMap;
-import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -74,9 +72,7 @@
 import org.apache.pulsar.broker.loadbalance.LeaderBroker;
 import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
-import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
-import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
@@ -206,10 +202,9 @@ protected void cleanup() throws Exception {
     }
 
     @BeforeMethod
-    protected void initializeState() throws IllegalAccessException {
+    protected void initializeState() throws PulsarAdminException {
+        admin.namespaces().unload("public/default");
         reset(primaryLoadManager, secondaryLoadManager);
-        cleanTableView(channel1);
-        cleanTableView(channel2);
     }
 
     @Test
@@ -561,20 +556,20 @@ public void testGetMetrics() throws Exception {
             FieldUtils.writeDeclaredField(unloadCounter, "loadStd", 0.3, true);
             FieldUtils.writeDeclaredField(unloadCounter, "breakdownCounters", Map.of(
                     Success, new LinkedHashMap<>() {{
-                        put(Overloaded, new MutableLong(1));
-                        put(Underloaded, new MutableLong(2));
+                        put(Overloaded, new AtomicLong(1));
+                        put(Underloaded, new AtomicLong(2));
                     }},
                     Skip, new LinkedHashMap<>() {{
-                        put(Balanced, new MutableLong(3));
-                        put(NoBundles, new MutableLong(4));
-                        put(CoolDown, new MutableLong(5));
-                        put(OutDatedData, new MutableLong(6));
-                        put(NoLoadData, new MutableLong(7));
-                        put(NoBrokers, new MutableLong(8));
-                        put(Unknown, new MutableLong(9));
+                        put(Balanced, new AtomicLong(3));
+                        put(NoBundles, new AtomicLong(4));
+                        put(CoolDown, new AtomicLong(5));
+                        put(OutDatedData, new AtomicLong(6));
+                        put(NoLoadData, new AtomicLong(7));
+                        put(NoBrokers, new AtomicLong(8));
+                        put(Unknown, new AtomicLong(9));
                     }},
                     Failure, Map.of(
-                            Unknown, new MutableLong(10))
+                            Unknown, new AtomicLong(10))
             ), true);
             unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress()));
         }
@@ -722,15 +717,6 @@ public void initialize(PulsarService pulsar) {
 
     }
 
-    private static void cleanTableView(ServiceUnitStateChannel channel)
-            throws IllegalAccessException {
-        var tv = (TableViewImpl)
-                FieldUtils.readField(channel, "tableview", true);
-        var cache = (ConcurrentMap)
-                FieldUtils.readField(tv, "data", true);
-        cache.clear();
-    }
-
     private void setPrimaryLoadManager() throws IllegalAccessException {
         ExtensibleLoadManagerWrapper wrapper =
                 (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java
index 75ef913b8a851..6a2ae1cc562cc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java
@@ -19,6 +19,10 @@
 package org.apache.pulsar.broker.loadbalance.extensions.manager;
 
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
@@ -32,6 +36,9 @@
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.testng.annotations.Test;
 
@@ -41,10 +48,13 @@ public class UnloadManagerTest {
 
     @Test
     public void testEventPubFutureHasException() {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(FutureUtil.failedFuture(new Exception("test")),
-                        "bundle-1", 10, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 10, TimeUnit.SECONDS);
 
         assertTrue(future.isCompletedExceptionally());
         try {
@@ -53,14 +63,18 @@ public void testEventPubFutureHasException() {
         } catch (Exception ex) {
             assertEquals(ex.getCause().getMessage(), "test");
         }
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     @Test
     public void testTimeout() throws IllegalAccessException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 3, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 3, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -73,14 +87,18 @@ public void testTimeout() throws IllegalAccessException {
         }
 
         assertEquals(inFlightUnloadRequestMap.size(), 0);
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     @Test
     public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 5, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 5, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -109,10 +127,11 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int
                 new ServiceUnitStateData(ServiceUnitState.Free, "broker-1", VERSION_ID_INIT), null);
         assertEquals(inFlightUnloadRequestMap.size(), 0);
         future.get();
+        assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 1);
 
         // Success with Owned state.
         future = manager.waitAsync(CompletableFuture.completedFuture(null),
-                "bundle-1", 5, TimeUnit.SECONDS);
+                "bundle-1", unloadDecision, 5, TimeUnit.SECONDS);
         inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -121,14 +140,19 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int
                 new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), null);
         assertEquals(inFlightUnloadRequestMap.size(), 0);
         future.get();
+
+        assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 2);
     }
 
     @Test
     public void testFailedStage() throws IllegalAccessException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 5, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 5, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -146,14 +170,18 @@ public void testFailedStage() throws IllegalAccessException {
         }
 
         assertEquals(inFlightUnloadRequestMap.size(), 0);
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     @Test
     public void testClose() throws IllegalAccessException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 5, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision,5, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
         assertEquals(inFlightUnloadRequestMap.size(), 1);
         manager.close();
@@ -165,6 +193,7 @@ public void testClose() throws IllegalAccessException {
         } catch (Exception ex) {
             assertTrue(ex.getCause() instanceof IllegalStateException);
         }
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     private Map> getInFlightUnloadRequestMap(UnloadManager manager)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 3955f1ed9af2e..a89f33bb98737 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -45,6 +45,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
@@ -64,6 +65,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
@@ -351,19 +353,19 @@ public void startTableView() throws LoadDataStoreException {
 
     @Test
     public void testEmptyBrokerLoadData() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(NoBrokers);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBrokers).get(), 1);
     }
 
     @Test
     public void testEmptyTopBundlesLoadData() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
@@ -373,21 +375,20 @@ public void testEmptyTopBundlesLoadData() {
         brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  20));
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(NoLoadData);
-        expected.setLoadAvg(0.39999999999999997);
-        expected.setLoadStd(0.35590260840104376);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoLoadData).get(), 1);
+        assertEquals(counter.getLoadAvg(), 0.39999999999999997);
+        assertEquals(counter.getLoadStd(), 0.35590260840104376);
     }
 
     @Test
     public void testOutDatedLoadData() throws IllegalAccessException {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        assertEquals(res.getUnloads().size(), 2);
+        assertEquals(res.size(), 2);
 
 
         FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1").get(), "updatedAt", 0, true);
@@ -398,16 +399,14 @@ public void testOutDatedLoadData() throws IllegalAccessException {
 
 
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(OutDatedData);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(OutDatedData).get(), 1);
     }
 
     @Test
     public void testRecentlyUnloadedBrokers() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
 
         Map recentlyUnloadedBrokers = new HashMap<>();
@@ -416,32 +415,27 @@ public void testRecentlyUnloadedBrokers() {
         recentlyUnloadedBrokers.put("broker1", oldTS);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers);
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
 
         var now = System.currentTimeMillis();
         recentlyUnloadedBrokers.put("broker1", now);
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers);
 
-        expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(CoolDown);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(CoolDown).get(), 1);
     }
 
     @Test
     public void testRecentlyUnloadedBundles() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         Map recentlyUnloadedBundles = new HashMap<>();
         var now = System.currentTimeMillis();
@@ -451,35 +445,34 @@ public void testRecentlyUnloadedBundles() {
         recentlyUnloadedBundles.put(bundleD2, now);
         var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(NoBundles);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testGetAvailableBrokersFailed() {
-        TransferShedder transferShedder = new TransferShedder();
+        var pulsar = getMockPulsar();
+        UnloadCounter counter = new UnloadCounter();
+        AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper);
         var ctx = setupContext();
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(Unknown);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test(timeOut = 30 * 1000)
     public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
-
-
-        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
+        var pulsar = getMockPulsar();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper));
 
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
@@ -487,33 +480,27 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
         IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy);
         FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true);
 
-        // Test transfer to a has isolation policies broker.
         setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
                 Set.of("broker5"), Set.of(), Set.of(), 1);
         var ctx = setupContext();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
 
         // Test unload a has isolation policies broker.
         ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false);
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        expected = new UnloadDecision();
-        unloads = expected.getUnloads();
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.empty()));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+
+        expected = new HashSet<>();
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.empty()),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     public BrokerLookupData getLookupData() {
@@ -590,7 +577,8 @@ private PulsarService getMockPulsar() {
     @Test
     public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
         var pulsar = getMockPulsar();
-        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
+        var counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper);
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
         doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
@@ -603,29 +591,26 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(NoBundles);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+
 
         doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any());
         var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected2 = new UnloadDecision();
-        var unloads = expected2.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        expected2.setLabel(Success);
-        expected2.setReason(Overloaded);
-        expected2.setLoadAvg(setupLoadAvg);
-        expected2.setLoadStd(setupLoadStd);
-        assertEquals(res2, expected2);
+        var expected2 = new HashSet<>();
+        expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(res, expected2);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testTargetStd() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
@@ -641,17 +626,16 @@ public void testTargetStd() {
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(Balanced);
-        expected.setLoadAvg(0.2000000063578288);
-        expected.setLoadStd(0.08164966587949089);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(Balanced).get(), 1);
+        assertEquals(counter.getLoadAvg(), 0.2000000063578288);
+        assertEquals(counter.getLoadStd(), 0.08164966587949089);
     }
 
     @Test
     public void testSingleTopBundlesLoadData() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
         topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1));
@@ -661,38 +645,35 @@ public void testSingleTopBundlesLoadData() {
         topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(NoBundles);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
 
     @Test
     public void testTargetStdAfterTransfer() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
         brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(0.26400000000000007);
-        expected.setLoadStd(0.27644891028904417);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.26400000000000007);
+        assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
     @Test
     public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
@@ -705,64 +686,57 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Underloaded);
-        expected.setLoadAvg(0.26400000000000007);
-        expected.setLoadStd(0.27644891028904417);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Underloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.26400000000000007);
+        assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
     @Test
     public void testMaxNumberOfTransfersPerShedderCycle() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         ctx.brokerConfiguration()
                 .setLoadBalancerMaxNumberOfBrokerTransfersPerCycle(10);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testRemainingTopBundles() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
         topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testRandomLoad() throws IllegalAccessException {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         for (int i = 0; i < 5; i++) {
             var ctx = setupContext(10);
             var conf = ctx.brokerConfiguration();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
index 73d4eb1f18bfb..7d9cf556360e4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
@@ -28,23 +28,29 @@
 import static org.mockito.Mockito.verify;
 
 import com.google.common.collect.Lists;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.client.util.ExecutorProvider;
+import org.apache.pulsar.common.stats.Metrics;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 @Test(groups = "broker")
 public class UnloadSchedulerTest {
@@ -70,23 +76,28 @@ public void tearDown() {
 
     @Test(timeOut = 30 * 1000)
     public void testExecuteSuccess() {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         BrokerRegistry registry = context.brokerRegistry();
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
+        PulsarService pulsar = mock(PulsarService.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync();
         doReturn(CompletableFuture.completedFuture(Lists.newArrayList("broker-1", "broker-2")))
                 .when(registry).getAvailableBrokersAsync();
         doReturn(CompletableFuture.completedFuture(null)).when(channel).publishUnloadEventAsync(any());
         doReturn(CompletableFuture.completedFuture(null)).when(unloadManager)
-                .waitAsync(any(), any(), anyLong(), any());
+                .waitAsync(any(), any(), any(), anyLong(), any());
         UnloadDecision decision = new UnloadDecision();
         Unload unload = new Unload("broker-1", "bundle-1");
-        decision.getUnloads().put("broker-1", unload);
-        doReturn(decision).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
+        decision.setUnload(unload);
+        decision.setLabel(UnloadDecision.Label.Success);
+        doReturn(Set.of(decision)).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
 
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
 
         scheduler.execute();
 
@@ -94,7 +105,7 @@ public void testExecuteSuccess() {
 
         // Test empty unload.
         UnloadDecision emptyUnload = new UnloadDecision();
-        doReturn(emptyUnload).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
+        doReturn(Set.of(emptyUnload)).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
 
         scheduler.execute();
 
@@ -103,26 +114,29 @@ public void testExecuteSuccess() {
 
     @Test(timeOut = 30 * 1000)
     public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedException {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         BrokerRegistry registry = context.brokerRegistry();
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
+        PulsarService pulsar = mock(PulsarService.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync();
         doAnswer(__ -> CompletableFuture.supplyAsync(() -> {
                 try {
                     // Delay 5 seconds to finish.
-                    TimeUnit.SECONDS.sleep(5);
+                    TimeUnit.SECONDS.sleep(1);
                 } catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
                 return Lists.newArrayList("broker-1", "broker-2");
             }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync();
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
-
-        ExecutorService executorService = Executors.newFixedThreadPool(10);
-        CountDownLatch latch = new CountDownLatch(10);
-        for (int i = 0; i < 10; i++) {
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
+        ExecutorService executorService = Executors.newFixedThreadPool(5);
+        CountDownLatch latch = new CountDownLatch(5);
+        for (int i = 0; i < 5; i++) {
             executorService.execute(() -> {
                 scheduler.execute();
                 latch.countDown();
@@ -130,18 +144,21 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio
         }
         latch.await();
 
-        verify(registry, times(1)).getAvailableBrokersAsync();
+        verify(registry, times(5)).getAvailableBrokersAsync();
     }
 
     @Test(timeOut = 30 * 1000)
     public void testDisableLoadBalancer() {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         context.brokerConfiguration().setLoadBalancerEnabled(false);
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
-
+        PulsarService pulsar = mock(PulsarService.class);
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
         scheduler.execute();
 
         verify(channel, times(0)).isChannelOwnerAsync();
@@ -155,12 +172,16 @@ public void testDisableLoadBalancer() {
 
     @Test(timeOut = 30 * 1000)
     public void testNotChannelOwner() {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         context.brokerConfiguration().setLoadBalancerEnabled(false);
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
+        PulsarService pulsar = mock(PulsarService.class);
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
         doReturn(CompletableFuture.completedFuture(false)).when(channel).isChannelOwnerAsync();
 
         scheduler.execute();

From 13d78dcbdb59907d2541e7ee8fe96101fd441b12 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Sun, 19 Mar 2023 11:16:34 +0800
Subject: [PATCH 085/404] [improve][broker] PIP-192: Make split admin API
 functional (#19773)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691, this PR makes the split admin API functional.

### Modifications

* Make the split admin API functional when enabling the `ExtensibleLoadManager`.
* Support more split algorithm.
---
 .../extensions/ExtensibleLoadManagerImpl.java | 60 +++++++++++++++++-
 .../channel/ServiceUnitStateChannelImpl.java  | 34 +++++++++--
 .../channel/ServiceUnitStateData.java         | 17 ++++--
 .../loadbalance/extensions/models/Split.java  |  7 +++
 ...faultNamespaceBundleSplitStrategyImpl.java |  2 +-
 .../broker/namespace/NamespaceService.java    |  9 +--
 .../pulsar/broker/web/PulsarWebResource.java  |  7 ++-
 .../pulsar/common/naming/NamespaceBundle.java |  2 +-
 ...pecifiedPositionsBundleSplitAlgorithm.java |  2 +-
 .../ExtensibleLoadManagerImplTest.java        | 61 +++++++++++++++++++
 .../channel/ServiceUnitStateChannelTest.java  | 11 ++--
 .../extensions/channel/models/SplitTest.java  | 22 ++++++-
 .../scheduler/SplitSchedulerTest.java         |  5 +-
 ...faultNamespaceBundleSplitStrategyTest.java | 17 +++---
 14 files changed, 216 insertions(+), 40 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 486c32153589f..fedcad4d009b2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -18,10 +18,11 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions;
 
+import static java.lang.String.format;
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower;
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -56,7 +57,9 @@
 import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
@@ -71,11 +74,15 @@
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.stats.Metrics;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 
@@ -406,7 +413,7 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                     }
                     Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker);
                     UnloadDecision unloadDecision =
-                            new UnloadDecision(unload, Success, Admin);
+                            new UnloadDecision(unload, UnloadDecision.Label.Success, UnloadDecision.Reason.Admin);
                     return unloadAsync(unloadDecision,
                             conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
                 });
@@ -421,6 +428,53 @@ private CompletableFuture unloadAsync(UnloadDecision unloadDecision,
                 .thenRun(() -> unloadCounter.updateUnloadBrokerCount(1));
     }
 
+    public CompletableFuture splitNamespaceBundleAsync(ServiceUnitId bundle,
+                                                             NamespaceBundleSplitAlgorithm splitAlgorithm,
+                                                             List boundaries) {
+        final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle.toString());
+        final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle.toString());
+        NamespaceBundle namespaceBundle =
+                pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange);
+        return pulsar.getNamespaceService().getSplitBoundary(namespaceBundle, splitAlgorithm, boundaries)
+                .thenCompose(splitBundlesPair -> {
+                    if (splitBundlesPair == null) {
+                        String msg = format("Bundle %s not found under namespace", namespaceBundle);
+                        log.error(msg);
+                        return FutureUtil.failedFuture(new IllegalStateException(msg));
+                    }
+
+                    return getOwnershipAsync(Optional.empty(), bundle)
+                            .thenCompose(brokerOpt -> {
+                                if (brokerOpt.isEmpty()) {
+                                    String msg = String.format("Namespace bundle: %s is not owned by any broker.",
+                                            bundle);
+                                    log.warn(msg);
+                                    throw new IllegalStateException(msg);
+                                }
+                                String sourceBroker = brokerOpt.get();
+                                SplitDecision splitDecision = new SplitDecision();
+                                List splitBundles = splitBundlesPair.getRight();
+                                Map> splitServiceUnitToDestBroker = new HashMap<>();
+                                splitBundles.forEach(splitBundle -> splitServiceUnitToDestBroker
+                                        .put(splitBundle.getBundleRange(), Optional.empty()));
+                                splitDecision.setSplit(
+                                        new Split(bundle.toString(), sourceBroker, splitServiceUnitToDestBroker));
+                                splitDecision.setLabel(Success);
+                                splitDecision.setReason(Admin);
+                                return splitAsync(splitDecision,
+                                        conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
+                            });
+                });
+    }
+
+    private CompletableFuture splitAsync(SplitDecision decision,
+                                               long timeout,
+                                               TimeUnit timeoutUnit) {
+        Split split = decision.getSplit();
+        CompletableFuture future = serviceUnitStateChannel.publishSplitEventAsync(split);
+        return splitManager.waitAsync(future, decision.getSplit().serviceUnit(), decision, timeout, timeoutUnit);
+    }
+
     @Override
     public void close() throws PulsarServerException {
         if (!this.started) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 791d02649baf1..ca9e55923e799 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -87,6 +87,7 @@
 import org.apache.pulsar.client.api.TableView;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceBundles;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicDomain;
@@ -533,7 +534,8 @@ public CompletableFuture publishSplitEventAsync(Split split) {
         eventCounters.get(eventType).getTotal().incrementAndGet();
         String serviceUnit = split.serviceUnit();
         ServiceUnitStateData next =
-                new ServiceUnitStateData(Splitting, null, split.sourceBroker(), getNextVersionId(serviceUnit));
+                new ServiceUnitStateData(Splitting, null, split.sourceBroker(),
+                        split.splitServiceUnitToDestBroker(), getNextVersionId(serviceUnit));
         return pubAsync(serviceUnit, next).whenComplete((__, ex) -> {
             if (ex != null) {
                 eventCounters.get(eventType).getFailure().incrementAndGet();
@@ -784,22 +786,40 @@ private CompletableFuture closeServiceUnit(String serviceUnit) {
     }
 
     private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnitStateData data) {
-        // Write the child ownerships to BSC.
+        // Write the child ownerships to channel.
         long startTime = System.nanoTime();
         NamespaceService namespaceService = pulsar.getNamespaceService();
         NamespaceBundleFactory bundleFactory = namespaceService.getNamespaceBundleFactory();
         NamespaceBundle bundle = getNamespaceBundle(serviceUnit);
         CompletableFuture completionFuture = new CompletableFuture<>();
+        Map> bundleToDestBroker = data.splitServiceUnitToDestBroker();
+        List boundaries = null;
+        NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm =
+                namespaceService.getNamespaceBundleSplitAlgorithmByName(
+                        config.getDefaultNamespaceBundleSplitAlgorithm());
+        if (bundleToDestBroker != null && bundleToDestBroker.size() == 2) {
+            Set boundariesSet = new HashSet<>();
+            String namespace = bundle.getNamespaceObject().toString();
+            bundleToDestBroker.forEach((bundleRange, destBroker) -> {
+                NamespaceBundle subBundle = bundleFactory.getBundle(namespace, bundleRange);
+                boundariesSet.add(subBundle.getKeyRange().lowerEndpoint());
+                boundariesSet.add(subBundle.getKeyRange().upperEndpoint());
+            });
+            boundaries = new ArrayList<>(boundariesSet);
+            nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_ALGO;
+        }
         final AtomicInteger counter = new AtomicInteger(0);
-        this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, bundle, serviceUnit, data,
-                counter, startTime, completionFuture);
+        this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, nsBundleSplitAlgorithm,
+                bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture);
         return completionFuture;
     }
 
     @VisibleForTesting
     protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                                                 NamespaceBundleFactory bundleFactory,
+                                                NamespaceBundleSplitAlgorithm algorithm,
                                                 NamespaceBundle bundle,
+                                                List boundaries,
                                                 String serviceUnit,
                                                 ServiceUnitStateData data,
                                                 AtomicInteger counter,
@@ -807,7 +827,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                                                 CompletableFuture completionFuture) {
         CompletableFuture> updateFuture = new CompletableFuture<>();
 
-        pulsar.getNamespaceService().getSplitBoundary(bundle, null).thenAccept(splitBundlesPair -> {
+        namespaceService.getSplitBoundary(bundle, algorithm, boundaries)
+                .thenAccept(splitBundlesPair -> {
             // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper.
             if (splitBundlesPair == null) {
                 String msg = format("Bundle %s not found under namespace", serviceUnit);
@@ -877,7 +898,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                 log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit",
                         counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex);
                 pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory,
-                                bundle, serviceUnit, data, counter, startTime, completionFuture), 100, MILLISECONDS);
+                        algorithm, bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture),
+                        100, MILLISECONDS);
             } else if (throwable instanceof IllegalArgumentException) {
                 completionFuture.completeExceptionally(throwable);
             } else {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
index c26dce83a4434..52c7e27d65033 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
@@ -18,7 +18,9 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.channel;
 
+import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import org.apache.commons.lang3.StringUtils;
 
 /**
@@ -27,7 +29,8 @@
  */
 
 public record ServiceUnitStateData(
-        ServiceUnitState state, String dstBroker, String sourceBroker, boolean force, long timestamp, long versionId) {
+        ServiceUnitState state, String dstBroker, String sourceBroker,
+        Map> splitServiceUnitToDestBroker, boolean force, long timestamp, long versionId) {
 
     public ServiceUnitStateData {
         Objects.requireNonNull(state);
@@ -36,16 +39,22 @@ public record ServiceUnitStateData(
         }
     }
 
+    public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker,
+                                Map> splitServiceUnitToDestBroker, long versionId) {
+        this(state, dstBroker, sourceBroker, splitServiceUnitToDestBroker, false,
+                System.currentTimeMillis(), versionId);
+    }
+
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) {
-        this(state, dstBroker, sourceBroker, false, System.currentTimeMillis(), versionId);
+        this(state, dstBroker, sourceBroker, null, false, System.currentTimeMillis(), versionId);
     }
 
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) {
-        this(state, dstBroker, null, false, System.currentTimeMillis(), versionId);
+        this(state, dstBroker, null, null, false, System.currentTimeMillis(), versionId);
     }
 
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, boolean force, long versionId) {
-        this(state, dstBroker, null, force, System.currentTimeMillis(), versionId);
+        this(state, dstBroker, null, null, force, System.currentTimeMillis(), versionId);
     }
 
     public static ServiceUnitState state(ServiceUnitStateData data) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
index fcec8b0ae5541..ac9a36e6dbfd3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
@@ -30,5 +30,12 @@ public record Split(
 
     public Split {
         Objects.requireNonNull(serviceUnit);
+        if (splitServiceUnitToDestBroker != null && splitServiceUnitToDestBroker.size() != 2) {
+            throw new IllegalArgumentException("Split service unit should be split into 2 service units.");
+        }
+    }
+
+    public Split(String serviceUnit, String sourceBroker) {
+        this(serviceUnit, sourceBroker, null);
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
index e572fd4161bdb..da04a287f1ac5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -138,7 +138,7 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
                                     maxBundleBandwidth / LoadManagerShared.MIBI);
                         }
                         var decision = new SplitDecision();
-                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), new HashMap<>()));
+                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
                         decision.succeed(reason);
                         decisionCache.add(decision);
                         int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 899539e1db6a7..de18d50f3e144 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -822,7 +822,10 @@ public boolean isNamespaceBundleDisabled(NamespaceBundle bundle) throws Exceptio
     public CompletableFuture splitAndOwnBundle(NamespaceBundle bundle, boolean unload,
                                                      NamespaceBundleSplitAlgorithm splitAlgorithm,
                                                      List boundaries) {
-
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            return ExtensibleLoadManagerImpl.get(loadManager.get())
+                    .splitNamespaceBundleAsync(bundle, splitAlgorithm, boundaries);
+        }
         final CompletableFuture unloadFuture = new CompletableFuture<>();
         final AtomicInteger counter = new AtomicInteger(BUNDLE_SPLIT_RETRY_LIMIT);
         splitAndOwnBundleOnceAndRetry(bundle, unload, counter, unloadFuture, splitAlgorithm, boundaries);
@@ -963,10 +966,8 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle,
      * @return A pair, left is target namespace bundle, right is split bundles.
      */
     public CompletableFuture>> getSplitBoundary(
-            NamespaceBundle bundle, List boundaries) {
+            NamespaceBundle bundle, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm, List boundaries) {
         BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config);
-        NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm =
-                getNamespaceBundleSplitAlgorithmByName(config.getDefaultNamespaceBundleSplitAlgorithm());
         CompletableFuture> splitBoundary =
                 nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption);
         return splitBoundary.thenCompose(splitBoundaries -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
index 321a127ad97bd..c5713ebddaa2f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
@@ -636,9 +636,6 @@ protected CompletableFuture validateNamespaceBundleOwnershipAsy
         } catch (WebApplicationException wae) {
             return CompletableFuture.failedFuture(wae);
         }
-        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) {
-            return CompletableFuture.completedFuture(nsBundle);
-        }
         return validateBundleOwnershipAsync(nsBundle, authoritative, readOnly)
                 .thenApply(__ -> nsBundle);
     }
@@ -718,6 +715,10 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund
                         throw new RestException(Status.PRECONDITION_FAILED,
                                 "Failed to find ownership for ServiceUnit:" + bundle.toString());
                     }
+                    // If the load manager is extensible load manager, we don't need check the authoritative.
+                    if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) {
+                        return CompletableFuture.completedFuture(null);
+                    }
                     return nsService.isServiceUnitOwnedAsync(bundle)
                             .thenAccept(owned -> {
                                 if (!owned) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java
index 0b7e09fee2cec..78a80e077e42a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java
@@ -127,7 +127,7 @@ private static String getKey(NamespaceName nsname, Range keyRange) {
         return String.format("%s/0x%08x_0x%08x", nsname, keyRange.lowerEndpoint(), keyRange.upperEndpoint());
     }
 
-    Range getKeyRange() {
+    public Range getKeyRange() {
         return this.keyRange;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
index 6c341f185ca60..1fd9a85cfa782 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
@@ -54,4 +54,4 @@ public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSp
             return CompletableFuture.completedFuture(splitBoundaries);
         });
     }
-}
\ No newline at end of file
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 354a27a63c02a..d977e63330586 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -95,6 +95,7 @@
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.BundlesData;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.Policies;
 import org.apache.pulsar.common.policies.data.TenantInfo;
@@ -372,6 +373,66 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle)
         assertFalse(otherLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
     }
 
+    @Test(timeOut = 30 * 1000)
+    public void testSplitBundleAdminAPI() throws Exception {
+        String namespace = "public/default";
+        String topic = "persistent://" + namespace + "/test-split";
+        admin.topics().createPartitionedTopic(topic, 10);
+        BundlesData bundles = admin.namespaces().getBundles(namespace);
+        int numBundles = bundles.getNumBundles();
+        var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList();
+
+        String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1);
+
+        long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2;
+
+        admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null);
+
+        BundlesData bundlesData = admin.namespaces().getBundles(namespace);
+        assertEquals(bundlesData.getNumBundles(), numBundles + 1);
+        String lowBundle = String.format("0x%08x", bundleRanges.get(0));
+        String midBundle = String.format("0x%08x", mid);
+        String highBundle = String.format("0x%08x", bundleRanges.get(1));
+        assertTrue(bundlesData.getBoundaries().contains(lowBundle));
+        assertTrue(bundlesData.getBoundaries().contains(midBundle));
+        assertTrue(bundlesData.getBoundaries().contains(highBundle));
+
+        // Test split bundle with invalid bundle range.
+        try {
+            admin.namespaces().splitNamespaceBundle(namespace, "invalid", true, null);
+            fail();
+        } catch (PulsarAdminException ex) {
+            assertTrue(ex.getMessage().contains("Invalid bundle range"));
+        }
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception {
+        String namespace = "public/default";
+        String topic = "persistent://" + namespace + "/test-split-with-specific-position";
+        admin.topics().createPartitionedTopic(topic, 10);
+        BundlesData bundles = admin.namespaces().getBundles(namespace);
+        int numBundles = bundles.getNumBundles();
+
+        var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList();
+
+        String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1);
+
+        long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2;
+        long splitPosition = mid + 100;
+
+        admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true,
+                "specified_positions_divide", List.of(bundleRanges.get(0), bundleRanges.get(1), splitPosition));
+
+        BundlesData bundlesData = admin.namespaces().getBundles(namespace);
+        assertEquals(bundlesData.getNumBundles(), numBundles + 1);
+        String lowBundle = String.format("0x%08x", bundleRanges.get(0));
+        String midBundle = String.format("0x%08x", splitPosition);
+        String highBundle = String.format("0x%08x", bundleRanges.get(1));
+        assertTrue(bundlesData.getBoundaries().contains(lowBundle));
+        assertTrue(bundlesData.getBoundaries().contains(midBundle));
+        assertTrue(bundlesData.getBoundaries().contains(highBundle));
+    }
 
     @Test
     public void testMoreThenOneFilter() throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index e0bd69fce64c7..c05c8ac741565 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -54,7 +54,6 @@
 import static org.testng.Assert.assertTrue;
 import static org.testng.AssertJUnit.assertNotNull;
 import java.lang.reflect.Field;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -297,7 +296,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel)
             }
         }
         try {
-            channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, Map.of()))
+            channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1))
                     .get(2, TimeUnit.SECONDS);
         } catch (ExecutionException e) {
             if (e.getCause() instanceof IllegalStateException) {
@@ -554,11 +553,15 @@ public void splitAndRetryTest() throws Exception {
             }
             // Call the real method
             reset(namespaceService);
+            doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
+                    .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
             return future;
         }).when(namespaceService).updateNamespaceBundles(any(), any());
         doReturn(namespaceService).when(pulsar1).getNamespaceService();
+        doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
+                .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
 
-        Split split = new Split(bundle, ownerAddr1.get(), new HashMap<>());
+        Split split = new Split(bundle, ownerAddr1.get());
         channel1.publishSplitEventAsync(split);
 
         waitUntilState(channel1, bundle, Deleted);
@@ -570,7 +573,7 @@ public void splitAndRetryTest() throws Exception {
         validateEventCounters(channel2, 0, 0, 0, 0, 0, 0);
         // Verify the retry count
         verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1))
-                .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), anyLong(), any());
+                .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), any(), any(), anyLong(), any());
 
         // Assert child bundle ownerships in the channels.
         String childBundle1 = "public/default/0x7fffffff_0xffffffff";
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
index 67c9555c88f01..1e23ea74dca5e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.channel.models;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -32,6 +33,7 @@ public class SplitTest {
     public void testConstructor() {
         Map> map = new HashMap<>();
         map.put("C", Optional.of("test"));
+        map.put("D", Optional.of("test"));
 
         Split split = new Split("A", "B", map);
         assertEquals(split.serviceUnit(), "A");
@@ -44,4 +46,22 @@ public void testNullBundle() {
         new Split(null, "A", Map.of());
     }
 
-}
\ No newline at end of file
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testInvalidSplitServiceUnitToDestBroker() {
+        Map> map = new HashMap<>();
+        map.put("C", Optional.of("test"));
+        map.put("D", Optional.of("test"));
+        map.put("E", Optional.of("test"));
+        new Split("A", "B", map);
+    }
+
+    @Test
+    public void testNullSplitServiceUnitToDestBroker() {
+        var split = new Split("A", "B");
+        assertEquals(split.serviceUnit(), "A");
+        assertEquals(split.sourceBroker(), "B");
+        assertNull(split.splitServiceUnitToDestBroker());
+    }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
index 7988aa413366f..f7726f8e7a1aa 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
@@ -27,7 +27,6 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -78,11 +77,11 @@ public void setUp() {
         doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any());
 
         decision1 = new SplitDecision();
-        decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+        decision1.setSplit(new Split(bundle1, broker));
         decision1.succeed(SplitDecision.Reason.MsgRate);
 
         decision2 = new SplitDecision();
-        decision2.setSplit(new Split(bundle2, broker, new HashMap<>()));
+        decision2.setSplit(new Split(bundle2, broker));
         decision2.succeed(SplitDecision.Reason.Sessions);
         Set decisions = Set.of(decision1, decision2);
         doReturn(decisions).when(strategy).findBundlesToSplit(any(), any());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
index 71606bb85a3fe..deff6fb00fafb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -28,7 +28,6 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
@@ -172,14 +171,14 @@ public void testMaxMsgRate() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.MsgRate);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.MsgRate);
 
                 assertEquals(actual, Set.of(decision1));
@@ -200,14 +199,14 @@ public void testMaxTopics() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.Topics);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.Topics);
 
                 assertEquals(actual, Set.of(decision1));
@@ -231,14 +230,14 @@ public void testMaxSessions() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.Sessions);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.Sessions);
 
                 assertEquals(actual, Set.of(decision1));
@@ -262,14 +261,14 @@ public void testMaxBandwidthMbytes() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.Bandwidth);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.Bandwidth);
 
                 assertEquals(actual, Set.of(decision1));

From f20dc93bc7d3defcc824857a558f2f78726f13a1 Mon Sep 17 00:00:00 2001
From: tison 
Date: Sun, 19 Mar 2023 12:13:10 +0800
Subject: [PATCH 086/404] [improve][misc] Add idea icon (#19855)

Signed-off-by: tison 
---
 .github/changes-filter.yaml |  6 +++---
 .idea/icon.svg              | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 .idea/icon.svg

diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml
index 3ec2fc22946da..be6faa957887d 100644
--- a/.github/changes-filter.yaml
+++ b/.github/changes-filter.yaml
@@ -3,13 +3,13 @@
 all:
   - '**'
 docs:
-  - 'site2/**'
-  - 'deployment/**'
-  - '.asf.yaml'
   - '*.md'
   - '**/*.md'
+  - '.asf.yaml'
   - '.github/changes-filter.yaml'
   - '.github/ISSUE_TEMPLATE/**'
+  - '.idea/**'
+  - 'deployment/**'
   - 'wiki/**'
 tests:
   - added|modified: '**/src/test/java/**/*.java'
diff --git a/.idea/icon.svg b/.idea/icon.svg
new file mode 100644
index 0000000000000..bf9b232def4a4
--- /dev/null
+++ b/.idea/icon.svg
@@ -0,0 +1,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file

From 3c8ab7bc1fe1851ca24b0f12063cf2442276f656 Mon Sep 17 00:00:00 2001
From: ran 
Date: Mon, 20 Mar 2023 10:16:59 +0800
Subject: [PATCH 087/404] [fix][test] Fix flaky MetadataStoreStatsTest 
 (#19829)

---
 .../service/ManagedLedgerCompressionTest.java |  2 +-
 .../broker/stats/MetadataStoreStatsTest.java  | 81 +++++++++++++------
 2 files changed, 58 insertions(+), 25 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
index 4991d0a75363d..c13c8bd9fdc94 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
@@ -50,7 +50,7 @@ protected void cleanup() throws Exception {
         super.internalCleanup();
     }
 
-    @Test(timeOut = 1000 * 20)
+    @Test(timeOut = 1000 * 30)
     public void testRestartBrokerEnableManagedLedgerInfoCompression() throws Exception {
         String topic = newTopicName();
         @Cleanup
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
index f4deddf0cec9e..8ae0242c6232a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
@@ -21,9 +21,13 @@
 import com.google.common.collect.Multimap;
 import java.io.ByteArrayOutputStream;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
 import org.apache.pulsar.broker.service.BrokerTestBase;
@@ -107,13 +111,17 @@ public void testMetadataStoreStats() throws Exception {
         Assert.assertTrue(opsLatency.size() > 1, metricsDebugMessage);
         Assert.assertTrue(putBytes.size() > 1, metricsDebugMessage);
 
+        Set expectedMetadataStoreName = new HashSet<>();
+        expectedMetadataStoreName.add(MetadataStoreConfig.METADATA_STORE);
+        expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE);
+
+        AtomicInteger matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : opsLatency) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertNotNull(m.tags.get("status"), metricsDebugMessage);
 
             if (m.tags.get("status").equals("success")) {
@@ -138,15 +146,19 @@ public void testMetadataStoreStats() throws Exception {
                 }
             }
         }
+        // Because the combination quantity between status(success, fail) and type(get, del, put) is 6.
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size() * 6);
+
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : putBytes) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
     }
 
     @Test
@@ -193,43 +205,64 @@ public void testBatchMetadataStoreMetrics() throws Exception {
         Assert.assertTrue(batchExecuteTime.size() > 0, metricsDebugMessage);
         Assert.assertTrue(opsPerBatch.size() > 0, metricsDebugMessage);
 
+        Set expectedMetadataStoreName = new HashSet<>();
+        expectedMetadataStoreName.add(MetadataStoreConfig.METADATA_STORE);
+        expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE);
+
+        AtomicInteger matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : executorQueueSize) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
+
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : opsWaiting) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
 
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : batchExecuteTime) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
 
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : opsPerBatch) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
     }
+
+    private boolean isExpectedLabel(String metadataStoreName, Set expectedLabel,
+                                    AtomicInteger expectedLabelCount) {
+        if (StringUtils.isEmpty(metadataStoreName)
+                || !expectedLabel.contains(metadataStoreName)) {
+            return false;
+        } else {
+            expectedLabelCount.incrementAndGet();
+            return true;
+        }
+    }
+
 }

From 7f5ba2bcbca2a712d11b02d3e2ef4dc2f3052a92 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Mon, 20 Mar 2023 10:29:50 +0800
Subject: [PATCH 088/404] [fix][broker] Fix create cluster with empty url
 (#19762)

---
 .../broker/admin/AdminApiClusterTest.java     | 34 +++++++++++++++++++
 .../pulsar/common/util/URIPreconditions.java  |  2 +-
 .../common/util/URIPreconditionsTest.java     |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java
index c37cc234b1213..28ebe39e0aa39 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java
@@ -22,6 +22,8 @@
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.fail;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
@@ -107,4 +109,36 @@ public void testDeleteExistFailureDomain() throws PulsarAdminException {
 
         admin.clusters().deleteFailureDomain(CLUSTER, domainName);
     }
+
+    @Test
+    public void testCreateCluster() throws PulsarAdminException {
+        List clusterDataList = new ArrayList<>();
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl("http://pulsar.app:8080")
+                .serviceUrlTls("")
+                .brokerServiceUrl("pulsar://pulsar.app:6650")
+                .brokerServiceUrlTls("")
+                .build());
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl("")
+                .serviceUrlTls("https://pulsar.app:8443")
+                .brokerServiceUrl("")
+                .brokerServiceUrlTls("pulsar+ssl://pulsar.app:6651")
+                .build());
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl("")
+                .serviceUrlTls("")
+                .brokerServiceUrl("")
+                .brokerServiceUrlTls("")
+                .build());
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl(null)
+                .serviceUrlTls(null)
+                .brokerServiceUrl(null)
+                .brokerServiceUrlTls(null)
+                .build());
+        for (int i = 0; i < clusterDataList.size(); i++) {
+            admin.clusters().createCluster("cluster-test-" + i, clusterDataList.get(i));
+        }
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java
index f68ed5e41a430..ea7fae2fdf0e9 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java
@@ -67,7 +67,7 @@ public static void checkURIIfPresent(@Nullable String uri,
     public static void checkURIIfPresent(@Nullable String uri,
                                          @Nonnull Predicate predicate,
                                          @Nullable String errorMessage) throws IllegalArgumentException {
-        if (uri == null) {
+        if (uri == null || uri.length() == 0) {
             return;
         }
         checkURI(uri, predicate, errorMessage);
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java
index d5809e8fdd0d3..f5b9e454e081d 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java
@@ -30,6 +30,7 @@ public void testCheckURI() {
         // normal
         checkURI("http://pulsar.apache.org", uri -> true);
         checkURI("http://pulsar.apache.org", uri -> Objects.equals(uri.getScheme(), "http"));
+        checkURI("", uri -> true);
         // illegal
         try {
             checkURI("pulsar.apache.org", uri -> Objects.equals(uri.getScheme(), "http"));
@@ -48,6 +49,7 @@ public void testCheckURI() {
     @Test
     public void testCheckURIIfPresent() {
         checkURIIfPresent(null, uri -> false);
+        checkURIIfPresent("", uri -> false);
         checkURIIfPresent("http://pulsar.apache.org", uri -> true);
         try {
             checkURIIfPresent("http/pulsar.apache.org", uri -> uri.getScheme() != null, "Error");

From 2637bda91d46f443762b22ac9963935725af2c8a Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Mon, 20 Mar 2023 16:39:09 +0800
Subject: [PATCH 089/404] [fix][meta] Follow up #19817, Fix race condition
 between ResourceLock update and invalidation (#19844)

---
 .../apache/pulsar/common/util/FutureUtil.java |  35 ++++++
 .../pulsar/common/util/FutureUtilTest.java    |  72 ++++++++++++
 .../coordination/impl/LockManagerImpl.java    |   2 +-
 .../coordination/impl/ResourceLockImpl.java   | 106 +++++++-----------
 4 files changed, 147 insertions(+), 68 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
index 162ef1e52ffd6..531769a1e452b 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -35,6 +36,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.annotation.concurrent.ThreadSafe;
 
 /**
  * This class is aimed at simplifying work with {@code CompletableFuture}.
@@ -166,6 +168,39 @@ public static Throwable unwrapCompletionException(Throwable ex) {
         }
     }
 
+    @ThreadSafe
+    public static class Sequencer {
+        private CompletableFuture sequencerFuture = CompletableFuture.completedFuture(null);
+        private final boolean allowExceptionBreakChain;
+
+        public Sequencer(boolean allowExceptionBreakChain) {
+            this.allowExceptionBreakChain = allowExceptionBreakChain;
+        }
+
+        public static  Sequencer create(boolean allowExceptionBreakChain) {
+            return new Sequencer<>(allowExceptionBreakChain);
+        }
+        public static  Sequencer create() {
+            return new Sequencer<>(false);
+        }
+
+        /**
+         * @throws NullPointerException NPE when param is null
+         */
+        public synchronized CompletableFuture sequential(Supplier> newTask) {
+            Objects.requireNonNull(newTask);
+            if (sequencerFuture.isDone()) {
+                if (sequencerFuture.isCompletedExceptionally() && allowExceptionBreakChain) {
+                    return sequencerFuture;
+                }
+                return sequencerFuture = newTask.get();
+            }
+            return sequencerFuture = allowExceptionBreakChain
+                    ? sequencerFuture.thenCompose(__ -> newTask.get())
+                    : sequencerFuture.exceptionally(ex -> null).thenCompose(__ -> newTask.get());
+        }
+    }
+
     /**
      * Creates a new {@link CompletableFuture} instance with timeout handling.
      *
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java
index 856ec49da6e9d..6df4494edf886 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java
@@ -21,6 +21,7 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
@@ -32,6 +33,7 @@
 import lombok.Cleanup;
 import org.assertj.core.util.Lists;
 import org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
@@ -178,4 +180,74 @@ public void testWaitForAny() {
             assertTrue(ex.getCause() instanceof RuntimeException);
         }
     }
+
+    @Test
+    public void testSequencer() {
+        int concurrentNum = 1000;
+        final ScheduledExecutorService executor = Executors.newScheduledThreadPool(concurrentNum);
+        final FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create();
+        // normal case -- allowExceptionBreakChain=false
+        final List list = new ArrayList<>();
+        final List> futures = new ArrayList<>();
+        for (int i = 0; i < concurrentNum; i++) {
+            int finalI = i;
+            futures.add(sequencer.sequential(() -> CompletableFuture.runAsync(() -> {
+                list.add(finalI);
+            }, executor)));
+        }
+        FutureUtil.waitForAll(futures).join();
+        for (int i = 0; i < list.size(); i++) {
+            Assert.assertEquals(list.get(i), (Integer) i);
+        }
+
+        // exception case -- allowExceptionBreakChain=false
+        final List list2 = new ArrayList<>();
+        final List> futures2 = new ArrayList<>();
+        for (int i = 0; i < concurrentNum; i++) {
+            int finalI = i;
+            futures2.add(sequencer.sequential(() -> CompletableFuture.runAsync(() -> {
+                if (finalI == 2) {
+                    throw new IllegalStateException();
+                }
+                list2.add(finalI);
+            }, executor)));
+        }
+        try {
+            FutureUtil.waitForAll(futures2).join();
+        } catch (Throwable ignore) {
+
+        }
+        for (int i = 0; i < concurrentNum - 1; i++) {
+            if (i >= 2) {
+                Assert.assertEquals(list2.get(i), i + 1);
+            } else {
+                Assert.assertEquals(list2.get(i), (Integer) i);
+            }
+        }
+        // allowExceptionBreakChain=true
+        final FutureUtil.Sequencer sequencer2 = FutureUtil.Sequencer.create(true);
+        final List list3 = new ArrayList<>();
+        final List> futures3 = new ArrayList<>();
+        for (int i = 0; i < concurrentNum; i++) {
+            int finalI = i;
+            futures3.add(sequencer2.sequential(() -> CompletableFuture.runAsync(() -> {
+                if (finalI == 2) {
+                    throw new IllegalStateException();
+                }
+                list3.add(finalI);
+            }, executor)));
+        }
+        try {
+            FutureUtil.waitForAll(futures3).join();
+        } catch (Throwable ignore) {
+
+        }
+        for (int i = 2; i < concurrentNum; i++) {
+            Assert.assertTrue(futures3.get(i).isCompletedExceptionally());
+        }
+
+        for (int i = 0; i < 2; i++) {
+            Assert.assertEquals(list3.get(i), (Integer) i);
+        }
+    }
 }
\ No newline at end of file
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
index ca768d38490cc..f5e7e0528bd7b 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
@@ -124,7 +124,7 @@ private void handleSessionEvent(SessionEvent se) {
             if (se == SessionEvent.SessionReestablished) {
                 log.info("Metadata store session has been re-established. Revalidating all the existing locks.");
                 for (ResourceLockImpl lock : locks.values()) {
-                    futures.add(lock.revalidate(lock.getValue(), true, true));
+                    futures.add(lock.silentRevalidateOnce());
                 }
 
             } else if (se == SessionEvent.Reconnected) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
index 5271a73249d80..93c994b2436b9 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
@@ -44,7 +44,7 @@ public class ResourceLockImpl implements ResourceLock {
     private long version;
     private final CompletableFuture expiredFuture;
     private boolean revalidateAfterReconnection = false;
-    private CompletableFuture pendingOperationFuture;
+    private final FutureUtil.Sequencer sequencer;
 
     private enum State {
         Init,
@@ -61,7 +61,7 @@ public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, Str
         this.path = path;
         this.version = -1;
         this.expiredFuture = new CompletableFuture<>();
-        this.pendingOperationFuture = CompletableFuture.completedFuture(null);
+        this.sequencer = FutureUtil.Sequencer.create();
         this.state = State.Init;
     }
 
@@ -74,11 +74,7 @@ public synchronized T getValue() {
     public synchronized CompletableFuture updateValue(T newValue) {
         // If there is an operation in progress, we're going to let it complete before attempting to
         // update the value
-        if (pendingOperationFuture.isDone()) {
-            pendingOperationFuture = CompletableFuture.completedFuture(null);
-        }
-
-        pendingOperationFuture = pendingOperationFuture.thenCompose(v -> {
+        return sequencer.sequential(() -> {
             synchronized (ResourceLockImpl.this) {
                 if (state != State.Valid) {
                     return CompletableFuture.failedFuture(
@@ -88,8 +84,6 @@ public synchronized CompletableFuture updateValue(T newValue) {
                 return acquire(newValue);
             }
         });
-
-        return pendingOperationFuture;
     }
 
     @Override
@@ -146,7 +140,7 @@ synchronized CompletableFuture acquire(T newValue) {
                 .thenRun(() -> result.complete(null))
                 .exceptionally(ex -> {
                     if (ex.getCause() instanceof LockBusyException) {
-                        revalidate(newValue, false, false)
+                        revalidate(newValue)
                                 .thenAccept(__ -> result.complete(null))
                                 .exceptionally(ex1 -> {
                                    result.completeExceptionally(ex1);
@@ -197,76 +191,54 @@ private CompletableFuture acquireWithNoRevalidation(T newValue) {
     }
 
     synchronized void lockWasInvalidated() {
-        if (state != State.Valid) {
-            // Ignore notifications while we're releasing the lock ourselves
-            return;
-        }
-
-        log.info("Lock on resource {} was invalidated", path);
-        revalidate(value, true, true)
-                .thenRun(() -> log.info("Successfully revalidated the lock on {}", path));
+        log.info("Lock on resource {} was invalidated. state {}", path, state);
+        silentRevalidateOnce();
     }
 
     synchronized CompletableFuture revalidateIfNeededAfterReconnection() {
         if (revalidateAfterReconnection) {
             revalidateAfterReconnection = false;
             log.warn("Revalidate lock at {} after reconnection", path);
-            return revalidate(value, true, true);
+            return silentRevalidateOnce();
         } else {
             return CompletableFuture.completedFuture(null);
         }
     }
 
-    synchronized CompletableFuture revalidate(T newValue, boolean trackPendingOperation,
-                                                    boolean revalidateAfterReconnection) {
-
-        final CompletableFuture trackFuture;
-
-        if (!trackPendingOperation) {
-            trackFuture = doRevalidate(newValue);
-        } else if (pendingOperationFuture.isDone()) {
-            pendingOperationFuture = doRevalidate(newValue);
-            trackFuture = pendingOperationFuture;
-        } else {
-            if (log.isDebugEnabled()) {
-                log.debug("Previous revalidating is not finished while revalidate newValue={}, value={}, version={}",
-                        newValue, value, version);
-            }
-            trackFuture = new CompletableFuture<>();
-            trackFuture.whenComplete((unused, throwable) -> {
-                doRevalidate(newValue).thenRun(() -> trackFuture.complete(null))
-                        .exceptionally(throwable1 -> {
-                            trackFuture.completeExceptionally(throwable1);
-                            return null;
-                        });
-            });
-            pendingOperationFuture = trackFuture;
-        }
-
-        trackFuture.exceptionally(ex -> {
-            synchronized (ResourceLockImpl.this) {
-                Throwable realCause = FutureUtil.unwrapCompletionException(ex);
-                if (!revalidateAfterReconnection || realCause instanceof BadVersionException
-                        || realCause instanceof LockBusyException) {
-                    log.warn("Failed to revalidate the lock at {}. Marked as expired. {}",
-                            path, realCause.getMessage());
-                    state = State.Released;
-                    expiredFuture.complete(null);
-                } else {
-                    // We failed to revalidate the lock due to connectivity issue
-                    // Continue assuming we hold the lock, until we can revalidate it, either
-                    // on Reconnected or SessionReestablished events.
-                    ResourceLockImpl.this.revalidateAfterReconnection = true;
-                    log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path,
-                            realCause.getMessage());
-                }
-            }
-            return null;
-        });
-        return trackFuture;
+    /**
+     * Revalidate the distributed lock if it is not released.
+     * This method is thread-safe and it will perform multiple re-validation operations in turn.
+     */
+    synchronized CompletableFuture silentRevalidateOnce() {
+        return sequencer.sequential(() -> revalidate(value))
+                .thenRun(() -> log.info("Successfully revalidated the lock on {}", path))
+                .exceptionally(ex -> {
+                    synchronized (ResourceLockImpl.this) {
+                        Throwable realCause = FutureUtil.unwrapCompletionException(ex);
+                        if (realCause instanceof BadVersionException || realCause instanceof LockBusyException) {
+                            log.warn("Failed to revalidate the lock at {}. Marked as expired. {}",
+                                    path, realCause.getMessage());
+                            state = State.Released;
+                            expiredFuture.complete(null);
+                        } else {
+                            // We failed to revalidate the lock due to connectivity issue
+                            // Continue assuming we hold the lock, until we can revalidate it, either
+                            // on Reconnected or SessionReestablished events.
+                            revalidateAfterReconnection = true;
+                            log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path,
+                                    realCause.getMessage());
+                        }
+                    }
+                    return null;
+                });
     }
 
-    private synchronized CompletableFuture doRevalidate(T newValue) {
+    private synchronized CompletableFuture revalidate(T newValue) {
+        // Since the distributed lock has been expired, we don't need to revalidate it.
+        if (state != State.Valid && state != State.Init) {
+            return CompletableFuture.failedFuture(
+                    new IllegalStateException("Lock was not in valid state: " + state));
+        }
         if (log.isDebugEnabled()) {
             log.debug("doRevalidate with newValue={}, version={}", newValue, version);
         }

From 5d2d61e298e337b93fb178da191bc41d311107c2 Mon Sep 17 00:00:00 2001
From: ran 
Date: Mon, 20 Mar 2023 17:41:20 +0800
Subject: [PATCH 090/404] [feat][txn] Transaction buffer snapshot writer reuse
 (#19641)

---
 .../SystemTopicTxnBufferSnapshotService.java  | 100 ++++++++++++++----
 ...SingleSnapshotAbortedTxnProcessorImpl.java |  21 ++--
 ...napshotSegmentAbortedTxnProcessorImpl.java |  45 ++++----
 .../SegmentAbortedTxnProcessorTest.java       |  19 +++-
 .../TopicTransactionBufferRecoverTest.java    |  17 +--
 .../broker/transaction/TransactionTest.java   |  85 ++++++++++++++-
 .../transaction/TransactionTestBase.java      |  20 ++++
 .../buffer/TransactionBufferCloseTest.java    |  79 ++++++++------
 8 files changed, 293 insertions(+), 93 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java
index a1b78d89a13eb..332d754cf97d2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java
@@ -21,63 +21,127 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
 
+@Slf4j
 public class SystemTopicTxnBufferSnapshotService {
 
-    protected final Map> clients;
+    protected final ConcurrentHashMap> clients;
     protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory;
 
     protected final Class schemaType;
     protected final EventType systemTopicType;
 
+    private final ConcurrentHashMap> refCountedWriterMap;
+
+    // The class ReferenceCountedWriter will maintain the reference count,
+    // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed.
+    public static class ReferenceCountedWriter {
+
+        private final AtomicLong referenceCount;
+        private final NamespaceName namespaceName;
+        private final CompletableFuture> future;
+        private final SystemTopicTxnBufferSnapshotService snapshotService;
+
+        public ReferenceCountedWriter(NamespaceName namespaceName,
+                                      CompletableFuture> future,
+                                      SystemTopicTxnBufferSnapshotService snapshotService) {
+            this.referenceCount = new AtomicLong(1);
+            this.namespaceName = namespaceName;
+            this.snapshotService = snapshotService;
+            this.future = future;
+            this.future.exceptionally(t -> {
+                        log.error("[{}] Failed to create TB snapshot writer.", namespaceName, t);
+                snapshotService.refCountedWriterMap.remove(namespaceName, this);
+                return null;
+            });
+        }
+
+        public CompletableFuture> getFuture() {
+            return future;
+        }
+
+        private synchronized boolean retain() {
+            return this.referenceCount.incrementAndGet() > 0;
+        }
+
+        public synchronized void release() {
+            if (this.referenceCount.decrementAndGet() == 0) {
+                snapshotService.refCountedWriterMap.remove(namespaceName, this);
+                future.thenAccept(writer -> {
+                    final String topicName = writer.getSystemTopicClient().getTopicName().toString();
+                    writer.closeAsync().exceptionally(t -> {
+                        if (t != null) {
+                            log.error("[{}] Failed to close TB snapshot writer.", topicName, t);
+                        } else {
+                            if (log.isDebugEnabled()) {
+                                log.debug("[{}] Success to close TB snapshot writer.", topicName);
+                            }
+                        }
+                        return null;
+                    });
+                });
+            }
+        }
+
+    }
+
     public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType systemTopicType,
                                                Class schemaType) {
         this.namespaceEventsSystemTopicFactory = new NamespaceEventsSystemTopicFactory(client);
         this.systemTopicType = systemTopicType;
         this.schemaType = schemaType;
         this.clients = new ConcurrentHashMap<>();
-    }
-
-    public CompletableFuture> createWriter(TopicName topicName) {
-        return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newWriterAsync);
+        this.refCountedWriterMap = new ConcurrentHashMap<>();
     }
 
     public CompletableFuture> createReader(TopicName topicName) {
-        return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newReaderAsync);
+        return getTransactionBufferSystemTopicClient(topicName.getNamespaceObject()).newReaderAsync();
     }
 
     public void removeClient(TopicName topicName, SystemTopicClientBase transactionBufferSystemTopicClient) {
         if (transactionBufferSystemTopicClient.getReaders().size() == 0
                 && transactionBufferSystemTopicClient.getWriters().size() == 0) {
-            clients.remove(topicName);
+            clients.remove(topicName.getNamespaceObject());
         }
     }
 
-    protected CompletableFuture> getTransactionBufferSystemTopicClient(TopicName topicName) {
+    public ReferenceCountedWriter getReferenceWriter(NamespaceName namespaceName) {
+        return refCountedWriterMap.compute(namespaceName, (k, v) -> {
+            if (v != null && v.retain()) {
+                return v;
+            } else {
+                return new ReferenceCountedWriter<>(namespaceName,
+                        getTransactionBufferSystemTopicClient(namespaceName).newWriterAsync(), this);
+            }
+        });
+    }
+
+    private SystemTopicClient getTransactionBufferSystemTopicClient(NamespaceName namespaceName) {
         TopicName systemTopicName = NamespaceEventsSystemTopicFactory
-                .getSystemTopicName(topicName.getNamespaceObject(), systemTopicType);
+                .getSystemTopicName(namespaceName, systemTopicType);
         if (systemTopicName == null) {
-            return FutureUtil.failedFuture(
-                    new PulsarClientException
-                            .InvalidTopicNameException("Can't create SystemTopicBaseTxnBufferSnapshotIndexService, "
-                            + "because the topicName is null!"));
+            throw new RuntimeException(new PulsarClientException.InvalidTopicNameException(
+                    "Can't get the TB system topic client for namespace " + namespaceName
+                            + " with type " + systemTopicType + "."));
         }
-        return CompletableFuture.completedFuture(clients.computeIfAbsent(systemTopicName,
+
+        return clients.computeIfAbsent(namespaceName,
                 (v) -> namespaceEventsSystemTopicFactory
-                        .createTransactionBufferSystemTopicClient(systemTopicName,
-                                this, schemaType)));
+                        .createTransactionBufferSystemTopicClient(systemTopicName, this, schemaType));
     }
 
     public void close() throws Exception {
-        for (Map.Entry> entry : clients.entrySet()) {
+        for (Map.Entry> entry : clients.entrySet()) {
             entry.getValue().close();
         }
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
index 87161e97512b9..5d582d564eadd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
@@ -28,6 +28,7 @@
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.collections4.map.LinkedMap;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
@@ -42,7 +43,7 @@
 @Slf4j
 public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor {
     private final PersistentTopic topic;
-    private final CompletableFuture> takeSnapshotWriter;
+    private final ReferenceCountedWriter takeSnapshotWriter;
     /**
      * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this
      * position have been deleted.
@@ -51,12 +52,14 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso
 
     private volatile long lastSnapshotTimestamps;
 
+    private volatile boolean isClosed = false;
+
     public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) {
         this.topic = topic;
         this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar()
                 .getTransactionBufferSnapshotServiceFactory()
-                .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName()));
-        this.takeSnapshotWriter.exceptionally((ex) -> {
+                .getTxnBufferSnapshotService().getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject());
+        this.takeSnapshotWriter.getFuture().exceptionally((ex) -> {
                     log.error("{} Failed to create snapshot writer", topic.getName());
                     topic.close();
                     return null;
@@ -132,7 +135,7 @@ public CompletableFuture recoverFromSnapshot() {
 
     @Override
     public CompletableFuture clearAbortedTxnSnapshot() {
-        return this.takeSnapshotWriter.thenCompose(writer -> {
+        return this.takeSnapshotWriter.getFuture().thenCompose(writer -> {
             TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot();
             snapshot.setTopicName(topic.getName());
             return writer.deleteAsync(snapshot.getTopicName(), snapshot);
@@ -141,7 +144,7 @@ public CompletableFuture clearAbortedTxnSnapshot() {
 
     @Override
     public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) {
-        return takeSnapshotWriter.thenCompose(writer -> {
+        return takeSnapshotWriter.getFuture().thenCompose(writer -> {
             TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot();
             snapshot.setTopicName(topic.getName());
             snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId());
@@ -175,8 +178,12 @@ public long getLastSnapshotTimestamps() {
     }
 
     @Override
-    public CompletableFuture closeAsync() {
-        return takeSnapshotWriter.thenCompose(SystemTopicClient.Writer::closeAsync);
+    public synchronized CompletableFuture closeAsync() {
+        if (!isClosed) {
+            isClosed = true;
+            takeSnapshotWriter.release();
+        }
+        return CompletableFuture.completedFuture(null);
     }
 
     private void closeReader(SystemTopicClient.Reader reader) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
index 7a9e0e1abedd9..751c03aff95a9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
@@ -43,6 +43,7 @@
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
@@ -442,10 +443,10 @@ public class PersistentWorker {
         private final PersistentTopic topic;
 
         //Persistent snapshot segment and index at the single thread.
-        private final CompletableFuture>
-                snapshotSegmentsWriterFuture;
-        private final CompletableFuture>
-                snapshotIndexWriterFuture;
+        private final ReferenceCountedWriter snapshotSegmentsWriter;
+        private final ReferenceCountedWriter snapshotIndexWriter;
+
+        private volatile boolean closed = false;
 
         private enum OperationState {
             None,
@@ -470,18 +471,20 @@ public enum OperationType {
 
         public PersistentWorker(PersistentTopic topic) {
             this.topic = topic;
-            this.snapshotSegmentsWriterFuture =  this.topic.getBrokerService().getPulsar()
+            this.snapshotSegmentsWriter = this.topic.getBrokerService().getPulsar()
                     .getTransactionBufferSnapshotServiceFactory()
-                    .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName()));
-            this.snapshotSegmentsWriterFuture.exceptionally(ex -> {
+                    .getTxnBufferSnapshotSegmentService()
+                    .getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject());
+            this.snapshotSegmentsWriter.getFuture().exceptionally(ex -> {
                         log.error("{} Failed to create snapshot index writer", topic.getName());
                         topic.close();
                         return null;
                     });
-            this.snapshotIndexWriterFuture =  this.topic.getBrokerService().getPulsar()
+            this.snapshotIndexWriter =  this.topic.getBrokerService().getPulsar()
                     .getTransactionBufferSnapshotServiceFactory()
-                    .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName()));
-            this.snapshotIndexWriterFuture.exceptionally((ex) -> {
+                    .getTxnBufferSnapshotIndexService()
+                    .getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject());
+            this.snapshotIndexWriter.getFuture().exceptionally((ex) -> {
                         log.error("{} Failed to create snapshot writer", topic.getName());
                         topic.close();
                         return null;
@@ -631,7 +634,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm
             transactionBufferSnapshotSegment.setPersistentPositionLedgerId(
                     abortedMarkerPersistentPosition.getLedgerId());
 
-            return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> {
+            return snapshotSegmentsWriter.getFuture().thenCompose(segmentWriter -> {
                 transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get());
                 return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment);
             }).thenCompose((messageId) -> {
@@ -668,7 +671,7 @@ private CompletableFuture deleteSnapshotSegment(List positio
             List> results = new ArrayList<>();
             for (PositionImpl positionNeedToDelete : positionNeedToDeletes) {
                 long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID();
-                CompletableFuture res = snapshotSegmentsWriterFuture
+                CompletableFuture res = snapshotSegmentsWriter.getFuture()
                         .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null))
                         .thenCompose(messageId -> {
                             if (log.isDebugEnabled()) {
@@ -695,7 +698,7 @@ private CompletableFuture deleteSnapshotSegment(List positio
 
         private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment) {
             TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes();
-            CompletableFuture res = snapshotIndexWriterFuture
+            CompletableFuture res = snapshotIndexWriter.getFuture()
                     .thenCompose((indexesWriter) -> {
                         snapshotIndexes.setIndexList(indexes.values().stream().toList());
                         snapshotIndexes.setSnapshot(snapshotSegment);
@@ -712,7 +715,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd
 
         private CompletableFuture clearSnapshotSegmentAndIndexes() {
             CompletableFuture res = persistentWorker.clearAllSnapshotSegments()
-                    .thenCompose((ignore) -> snapshotIndexWriterFuture
+                    .thenCompose((ignore) -> snapshotIndexWriter.getFuture()
                             .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null)))
                     .thenRun(() ->
                             log.debug("Successes to clear the snapshot segment and indexes for the topic [{}]",
@@ -747,7 +750,7 @@ private CompletableFuture clearAllSnapshotSegments() {
                                 Message message = reader.readNextAsync()
                                         .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS);
                                 if (topic.getName().equals(message.getValue().getTopicName())) {
-                                   snapshotSegmentsWriterFuture.get().write(message.getKey(), null);
+                                   snapshotSegmentsWriter.getFuture().get().write(message.getKey(), null);
                                 }
                             }
                             return CompletableFuture.completedFuture(null);
@@ -760,11 +763,13 @@ private CompletableFuture clearAllSnapshotSegments() {
            });
         }
 
-
-        CompletableFuture closeAsync() {
-            return CompletableFuture.allOf(
-                    this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync),
-                    this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync));
+        synchronized CompletableFuture closeAsync() {
+            if (!closed) {
+                closed = true;
+                snapshotSegmentsWriter.release();
+                snapshotIndexWriter.release();
+            }
+            return CompletableFuture.completedFuture(null);
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index ffc059de8e656..c157d7cf8c527 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -19,6 +19,8 @@
 package org.apache.pulsar.broker.transaction;
 
 import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
 import java.lang.reflect.Field;
 import java.util.LinkedList;
 import java.util.NavigableMap;
@@ -34,6 +36,7 @@
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
@@ -125,6 +128,7 @@ public void testPutAbortedTxnIntoProcessor() throws Exception {
         newProcessor.trimExpiredAbortedTxns();
         //4. Verify the two sealed segment will be deleted.
         Awaitility.await().untilAsserted(() -> verifyAbortedTxnIDAndSegmentIndex(newProcessor, 11, 4));
+        processor.closeAsync().get(5, TimeUnit.SECONDS);
     }
 
     private void waitTaskExecuteCompletely(AbortedTxnProcessor processor) throws Exception {
@@ -177,8 +181,11 @@ public void testFuturesCanCompleteWhenItIsCanceled() throws Exception {
                 new MutablePair<>(new CompletableFuture<>(), task)));
         try {
             processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS);
+            fail("The update index operation should fail.");
         } catch (Exception e) {
             Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException);
+        } finally {
+            processor.closeAsync().get(5, TimeUnit.SECONDS);
         }
     }
 
@@ -200,12 +207,13 @@ public void testClearSnapshotSegments() throws Exception {
         SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker worker =
                 (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) field.get(processor);
         Field indexWriteFutureField = SnapshotSegmentAbortedTxnProcessorImpl
-                .PersistentWorker.class.getDeclaredField("snapshotIndexWriterFuture");
+                .PersistentWorker.class.getDeclaredField("snapshotIndexWriter");
         indexWriteFutureField.setAccessible(true);
-        CompletableFuture> snapshotIndexWriterFuture =
-                (CompletableFuture>)
-                        indexWriteFutureField.get(worker);
-        snapshotIndexWriterFuture.get().close();
+        ReferenceCountedWriter snapshotIndexWriter =
+                (ReferenceCountedWriter) indexWriteFutureField.get(worker);
+        snapshotIndexWriter.release();
+        // After release, the writer should be closed, call close method again to make sure the writer was closed.
+        snapshotIndexWriter.getFuture().get().close();
         //3. Try to write a snapshot segment that will fail to update indexes.
         for (int j = 0; j < SEGMENT_SIZE; j++) {
             TxnID txnID = new TxnID(0, j);
@@ -233,6 +241,7 @@ public void testClearSnapshotSegments() throws Exception {
         //7. Verify the snapshot segments and index after clearing.
         verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 0);
         verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 1);
+        processor.closeAsync().get(5, TimeUnit.SECONDS);
     }
 
     private void verifySnapshotSegmentsSize(String topic, int size) throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java
index d4ddb26e014ca..2d6622571c033 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java
@@ -57,6 +57,7 @@
 import org.apache.pulsar.broker.service.AbstractTopic;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
@@ -602,11 +603,12 @@ public void testTransactionBufferRecoverThrowException() throws Exception {
                 mock(SystemTopicTxnBufferSnapshotService.class);
         SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class);
         SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class);
+        ReferenceCountedWriter refCounterWriter = mock(ReferenceCountedWriter.class);
+        doReturn(CompletableFuture.completedFuture(writer)).when(refCounterWriter).getFuture();
 
         doReturn(CompletableFuture.completedFuture(reader))
                 .when(systemTopicTxnBufferSnapshotService).createReader(any());
-        doReturn(CompletableFuture.completedFuture(writer))
-                .when(systemTopicTxnBufferSnapshotService).createWriter(any());
+        doReturn(refCounterWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any());
         TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory =
                 mock(TransactionBufferSnapshotServiceFactory.class);
         doReturn(systemTopicTxnBufferSnapshotService)
@@ -645,8 +647,9 @@ public void testTransactionBufferRecoverThrowException() throws Exception {
         originalTopic = (PersistentTopic) getPulsarServiceList().get(0)
                 .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get();
         // mock create writer fail
-        doReturn(FutureUtil.failedFuture(new PulsarClientException("test")))
-                .when(systemTopicTxnBufferSnapshotService).createWriter(any());
+        ReferenceCountedWriter failedCountedWriter = mock(ReferenceCountedWriter.class);
+        doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))).when(failedCountedWriter).getFuture();
+        doReturn(failedCountedWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any());
         checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceFactoryOriginal,
                 transactionBufferSnapshotServiceFactory, originalTopic, field, producer);
     }
@@ -703,7 +706,8 @@ public void testTransactionBufferIndexSystemTopic() throws Exception {
                 new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotIndexService();
 
         SystemTopicClient.Writer indexesWriter =
-                transactionBufferSnapshotIndexService.createWriter(TopicName.get(SNAPSHOT_INDEX)).get();
+                transactionBufferSnapshotIndexService.getReferenceWriter(
+                        TopicName.get(SNAPSHOT_INDEX).getNamespaceObject()).getFuture().get();
 
         SystemTopicClient.Reader indexesReader =
                 transactionBufferSnapshotIndexService.createReader(TopicName.get(SNAPSHOT_INDEX)).get();
@@ -764,7 +768,8 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception {
                 new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotSegmentService();
 
         SystemTopicClient.Writer
-                segmentWriter = transactionBufferSnapshotSegmentService.createWriter(snapshotSegmentTopicName).get();
+                segmentWriter = transactionBufferSnapshotSegmentService
+                .getReferenceWriter(snapshotSegmentTopicName.getNamespaceObject()).getFuture().get();
 
         // write two snapshot to snapshot segment topic
         TransactionBufferSnapshotSegment snapshot =
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
index 20aeac0ed648f..c3533e70cf8be 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
@@ -86,6 +86,7 @@
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
 import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
@@ -1532,8 +1533,10 @@ public void testTBRecoverChangeStateError() throws InterruptedException, Timeout
                 = mock(SystemTopicTxnBufferSnapshotService.class);
         SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class);
         when(writer.closeAsync()).thenReturn(CompletableFuture.completedFuture(null));
-        when(systemTopicTxnBufferSnapshotService.createWriter(any()))
-                .thenReturn(CompletableFuture.completedFuture(writer));
+        ReferenceCountedWriter refCounterWriter = mock(ReferenceCountedWriter.class);
+        doReturn(CompletableFuture.completedFuture(writer)).when(refCounterWriter).getFuture();
+        when(systemTopicTxnBufferSnapshotService.getReferenceWriter(any()))
+                .thenReturn(refCounterWriter);
         TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory =
                 mock(TransactionBufferSnapshotServiceFactory.class);
         when(transactionBufferSnapshotServiceFactory.getTxnBufferSnapshotService())
@@ -1676,4 +1679,82 @@ public void testDeleteNamespace() throws Exception {
         admin.namespaces().deleteNamespace(namespace, true);
     }
 
+
+    @Test(timeOut = 10_000)
+    public void testTBSnapshotWriter() throws Exception {
+        String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5);
+        admin.namespaces().createNamespace(namespace, 16);
+        String topic = namespace + "/test-create-snapshot-writer-failed";
+        int partitionCount = 20;
+        admin.topics().createPartitionedTopic(topic, partitionCount);
+
+        Class clazz = SystemTopicTxnBufferSnapshotService.class;
+        Field field = clazz.getDeclaredField("refCountedWriterMap");
+        field.setAccessible(true);
+        // inject a failed writer future
+        CompletableFuture> writerFuture = new CompletableFuture<>();
+        for (PulsarService pulsarService : pulsarServiceList) {
+            SystemTopicTxnBufferSnapshotService bufferSnapshotService =
+                    pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService();
+            ConcurrentHashMap writerMap1 =
+                    ((ConcurrentHashMap) field.get(bufferSnapshotService));
+            ReferenceCountedWriter failedCountedWriter =
+                    new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, bufferSnapshotService);
+            writerMap1.put(NamespaceName.get(namespace), failedCountedWriter);
+
+            SystemTopicTxnBufferSnapshotService segmentSnapshotService =
+                    pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotSegmentService();
+            ConcurrentHashMap writerMap2 =
+                    ((ConcurrentHashMap) field.get(segmentSnapshotService));
+            ReferenceCountedWriter failedCountedWriter2 =
+                    new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, segmentSnapshotService);
+            writerMap2.put(NamespaceName.get(namespace), failedCountedWriter2);
+
+            SystemTopicTxnBufferSnapshotService indexSnapshotService =
+                    pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotIndexService();
+            ConcurrentHashMap writerMap3 =
+                    ((ConcurrentHashMap) field.get(indexSnapshotService));
+            ReferenceCountedWriter failedCountedWriter3 =
+                    new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, indexSnapshotService);
+            writerMap3.put(NamespaceName.get(namespace), failedCountedWriter3);
+        }
+
+        CompletableFuture> producerFuture = pulsarClient.newProducer()
+                .topic(topic)
+                .sendTimeout(0, TimeUnit.SECONDS)
+                .createAsync();
+        getTopic("persistent://" + topic + "-partition-0");
+        Thread.sleep(3000);
+        // the producer shouldn't be created, because the transaction buffer snapshot writer future didn't finish.
+        assertFalse(producerFuture.isDone());
+
+        // The topic will be closed, because the transaction buffer snapshot writer future is failed,
+        // the failed writer future will be removed, the producer will be reconnected and work well.
+        writerFuture.completeExceptionally(new PulsarClientException.TopicTerminatedException("failed writer"));
+        Producer producer = producerFuture.get();
+
+        for (int i = 0; i < partitionCount * 2; i++) {
+            Transaction txn = pulsarClient.newTransaction()
+                    .withTransactionTimeout(1, TimeUnit.MINUTES).build().get();
+            producer.newMessage(txn).value("test".getBytes()).sendAsync();
+            txn.commit().get();
+        }
+        checkSnapshotPublisherCount(namespace, 1);
+        producer.close();
+        admin.topics().unload(topic);
+        checkSnapshotPublisherCount(namespace, 0);
+    }
+
+    private void getTopic(String topicName) {
+        Awaitility.await().atMost(5, TimeUnit.SECONDS).pollInterval(100, TimeUnit.MILLISECONDS)
+                .until(() -> {
+                    for (PulsarService pulsarService : pulsarServiceList) {
+                        if (pulsarService.getBrokerService().getTopicReference(topicName).isPresent()) {
+                            return true;
+                        }
+                    }
+                    return false;
+                });
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
index fd49354342fa0..f45eda8d21fbe 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
@@ -41,12 +41,17 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.SystemTopicNames;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.PublisherStats;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.tests.TestRetrySupport;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
 
 @Slf4j
 public abstract class TransactionTestBase extends TestRetrySupport {
@@ -223,4 +228,19 @@ protected void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin ad
             throws Exception {
         MockedPulsarServiceBaseTest.deleteNamespaceWithRetry(ns, force, admin, pulsarServiceList);
     }
+
+    public void checkSnapshotPublisherCount(String namespace, int expectCount) {
+        TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace),
+                SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT);
+        Awaitility.await()
+                .atMost(5, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> {
+                    List publisherStatsList =
+                            (List) admin.topics()
+                                    .getStats(snTopicName.getPartitionedTopicName()).getPublishers();
+                    Assert.assertEquals(publisherStatsList.size(), expectCount);
+                });
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
index cd3a14da5967d..e92cf29521e34 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
@@ -19,6 +19,9 @@
 package org.apache.pulsar.broker.transaction.buffer;
 
 import com.google.common.collect.Sets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.pulsar.broker.transaction.TransactionTestBase;
@@ -26,20 +29,13 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.naming.SystemTopicNames;
-import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.PublisherStats;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.awaitility.Awaitility;
-import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 /**
  * Transaction buffer close test.
@@ -71,49 +67,62 @@ public Object[][] isPartition() {
 
     @Test(timeOut = 10_000, dataProvider = "isPartition")
     public void deleteTopicCloseTransactionBufferTest(boolean isPartition) throws Exception {
-        int expectedCount = isPartition ? 30 : 1;
-        TopicName topicName = createAndLoadTopic(isPartition, expectedCount);
-        checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount);
+        int partitionCount = isPartition ? 30 : 1;
+        List topicNames = createAndLoadTopics(isPartition, partitionCount);
+        String namespaceName = topicNames.get(0).getNamespace();
+        checkSnapshotPublisherCount(namespaceName, 1);
+
+        for (int i = 0; i < topicNames.size(); i++) {
+            deleteTopic(isPartition, topicNames.get(i));
+            // When delete all topics of the namespace, the publisher count should be 0.
+            int expectCount = i == topicNames.size() - 1 ? 0 : 1;
+            checkSnapshotPublisherCount(namespaceName, expectCount);
+        }
+    }
+
+    private void deleteTopic(boolean isPartition, TopicName topicName) throws PulsarAdminException {
         if (isPartition) {
             admin.topics().deletePartitionedTopic(topicName.getPartitionedTopicName(), true);
         } else {
             admin.topics().delete(topicName.getPartitionedTopicName(), true);
         }
-        checkSnapshotPublisherCount(topicName.getNamespace(), 0);
     }
 
     @Test(timeOut = 10_000, dataProvider = "isPartition")
     public void unloadTopicCloseTransactionBufferTest(boolean isPartition) throws Exception {
-        int expectedCount = isPartition ? 30 : 1;
-        TopicName topicName = createAndLoadTopic(isPartition, expectedCount);
-        checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount);
-        admin.topics().unload(topicName.getPartitionedTopicName());
-        checkSnapshotPublisherCount(topicName.getNamespace(), 0);
+        int partitionCount = isPartition ? 30 : 1;
+        List topicNames = createAndLoadTopics(isPartition, partitionCount);
+        String namespaceName = topicNames.get(0).getNamespace();
+        checkSnapshotPublisherCount(namespaceName, 1);
+
+        for (int i = 0; i < topicNames.size(); i++) {
+            admin.topics().unload(topicNames.get(i).getPartitionedTopicName());
+            // When unload all topics of the namespace, the publisher count should be 0.
+            int expectCount = i == topicNames.size() - 1 ? 0 : 1;
+            checkSnapshotPublisherCount(namespaceName, expectCount);
+        }
     }
 
-    private TopicName createAndLoadTopic(boolean isPartition, int partitionCount)
+    private List createAndLoadTopics(boolean isPartition, int partitionCount)
             throws PulsarAdminException, PulsarClientException {
         String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5);
         admin.namespaces().createNamespace(namespace, 3);
-        String topic = namespace + "/tb-close-test-";
-        if (isPartition) {
-            admin.topics().createPartitionedTopic(topic, partitionCount);
-        }
-        pulsarClient.newProducer()
-                .topic(topic)
-                .sendTimeout(0, TimeUnit.SECONDS)
-                .create()
-                .close();
-        return TopicName.get(topic);
-    }
+        String topic = namespace + "/tb-close-test";
+        List topics = new ArrayList<>();
 
-    private void checkSnapshotPublisherCount(String namespace, int expectCount) throws PulsarAdminException {
-        TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace),
-                SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT);
-        List publisherStatsList =
-                (List) admin.topics()
-                        .getStats(snTopicName.getPartitionedTopicName()).getPublishers();
-        Assert.assertEquals(publisherStatsList.size(), expectCount);
+        for (int i = 0; i < 2; i++) {
+            String t = topic + "-" + i;
+            if (isPartition) {
+                admin.topics().createPartitionedTopic(t, partitionCount);
+            }
+            pulsarClient.newProducer()
+                    .topic(t)
+                    .sendTimeout(0, TimeUnit.SECONDS)
+                    .create()
+                    .close();
+            topics.add(TopicName.get(t));
+        }
+        return topics;
     }
 
 }

From c4abe7bd3c5fe59596decb0b6ce1badd40de0eeb Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Mon, 20 Mar 2023 23:09:49 +0800
Subject: [PATCH 091/404] [fix][sec] Upgrade dependency-check-maven and remove
 javax.el (#19764)

---
 pom.xml                                       | 20 ++++++++++++++++++-
 ...owasp-dependency-check-false-positives.xml | 10 ++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index dc27ed54274fb..257018b0b0598 100644
--- a/pom.xml
+++ b/pom.xml
@@ -291,7 +291,7 @@ flexible messaging model and an intuitive client API.
     0.1.4
     1.3
     0.4
-    8.0.1
+    8.1.2
     0.9.15
     1.6.1
     6.4.0
@@ -878,6 +878,24 @@ flexible messaging model and an intuitive client API.
         ${caffeine.version}
       
 
+      
+        org.bouncycastle
+        bcpkix-jdk15on
+        ${bouncycastle.version}
+      
+
+      
+        com.cronutils
+        cron-utils
+        ${cron-utils.version}
+        
+          
+            org.glassfish
+            javax.el
+          
+        
+      
+
       
         com.yahoo.athenz
         athenz-zts-java-client-core
diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml
index 21a3679c0d8f7..345be8f4d2c06 100644
--- a/src/owasp-dependency-check-false-positives.xml
+++ b/src/owasp-dependency-check-false-positives.xml
@@ -182,6 +182,16 @@
     CVE-2021-4277
   
 
+  
+    It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-25194 is a false positive.
+    CVE-2023-25194
+  
+
+  
+    It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-34917 is a false positive.
+    CVE-2022-34917
+  
+
   
     yaml_project is not used at all. Any CVEs reported for yaml_project are false positives.
     cpe:/a:yaml_project:yaml

From 164add6236eccb34f16683e86904f8d8d0b90c54 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 20 Mar 2023 13:55:02 -0500
Subject: [PATCH 092/404] [improve][proxy] Remove unnecessary executor
 callback; use assert (#19670)

### Motivation

The `DirectProxyHandler` and the `ProxyConnection` are run in the same event loop to prevent context switching. As such, we do not need to schedule an event onto an event loop that is in fact the same event loop. Further, scheduling on that event loop could have resulted in uncaught failures because the method was run without any error handling.

Additionally, we can use `assert` to verify that we are in the event loop. Netty makes extensive use of this paradigm, as described in this PR #19653. The primary benefit here is that we skip some unnecessary volatile reads when running the code in production.

### Modifications

* Replace `checkState` with `assert` in `ProxyConnection` class
* Remove unnecessary event execution in callback when starting up the `DirectProxyHandler`.

### Verifying this change

This change is covered by the assertions that are added.

### Does this pull request potentially affect one of the following parts:

This is a minor improvement that should not break anything.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/33
---
 .../apache/pulsar/proxy/server/ProxyConnection.java    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 5a53f6ec014a2..1e19d760c6d7b 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.proxy.server;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFutureListener;
@@ -380,7 +379,7 @@ private synchronized void completeConnect() throws PulsarClientException {
     }
 
     private void handleBrokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) {
-        checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop");
+        assert ctx.executor().inEventLoop();
         if (state == State.ProxyConnectingToBroker && ctx.channel().isOpen() && this.directProxyHandler == null) {
             this.directProxyHandler = directProxyHandler;
             state = State.ProxyConnectionToBroker;
@@ -401,7 +400,7 @@ private void handleBrokerConnected(DirectProxyHandler directProxyHandler, Comman
     }
 
     private void connectToBroker(InetSocketAddress brokerAddress) {
-        checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop");
+        assert ctx.executor().inEventLoop();
         DirectProxyHandler directProxyHandler = new DirectProxyHandler(service, this);
         directProxyHandler.connect(proxyToBrokerUrl, brokerAddress, protocolVersionToAdvertise);
     }
@@ -409,10 +408,13 @@ private void connectToBroker(InetSocketAddress brokerAddress) {
     public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) {
         try {
             final CommandConnected finalConnected = new CommandConnected().copyFrom(connected);
-            ctx.executor().execute(() -> handleBrokerConnected(directProxyHandler, finalConnected));
+            handleBrokerConnected(directProxyHandler, finalConnected);
         } catch (RejectedExecutionException e) {
             LOG.error("Event loop was already closed. Closing broker connection.", e);
             directProxyHandler.close();
+        } catch (AssertionError e) {
+            LOG.error("Failed assertion, closing direct proxy handler.", e);
+            directProxyHandler.close();
         }
     }
 

From c6de57c4b4510c5064c6e192df8fa9d51cede01d Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 20 Mar 2023 14:07:04 -0500
Subject: [PATCH 093/404] [improve][broker] Test AuthorizationService to cover
 proxyRoles behavior (#19845)

Relates to: #19455 #19830

### Motivation

When I added the requirement for the proxy to use a role in the `proxyRoles` set, I didn't add a test that checked the negative case. This new test was first added in #19830 with one small difference. The goal of this test is to ensure that authorization of the client role and the original role is handled correctly.

### Modifications

* Add new test class named `AuthorizationServiceTest`. We use `pass.proxy` and `fail.proxy` as proxy roles to simulate cases where the proxy's role passes and fails authorization, which is always possible.
* Add new mock authorization provider named `MockAuthorizationProvider`. The logic is to let any role that starts with `pass` be considered authorized.

### Verifying this change

This is a new test. It simply verifies the existing behavior to prevent future regressions.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: Skipping forked test since the new tests pass locally and there are no other modifications.
---
 .../AuthorizationServiceTest.java             | 135 ++++++++++++++++
 .../MockAuthorizationProvider.java            | 144 ++++++++++++++++++
 2 files changed, 279 insertions(+)
 create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java
 create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java

diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java
new file mode 100644
index 0000000000000..6f9dffa11b948
--- /dev/null
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authorization;
+
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+import java.util.HashSet;
+import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.NamespaceOperation;
+import org.apache.pulsar.common.policies.data.PolicyName;
+import org.apache.pulsar.common.policies.data.PolicyOperation;
+import org.apache.pulsar.common.policies.data.TenantOperation;
+import org.apache.pulsar.common.policies.data.TopicOperation;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class AuthorizationServiceTest {
+
+    AuthorizationService authorizationService;
+
+    @BeforeClass
+    void beforeClass() throws PulsarServerException {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthorizationEnabled(true);
+        // Consider both of these proxy roles to make testing more comprehensive
+        HashSet proxyRoles = new HashSet<>();
+        proxyRoles.add("pass.proxy");
+        proxyRoles.add("fail.proxy");
+        conf.setProxyRoles(proxyRoles);
+        conf.setAuthorizationProvider(MockAuthorizationProvider.class.getName());
+        authorizationService = new AuthorizationService(conf, null);
+    }
+
+    /**
+     * See {@link MockAuthorizationProvider} for the implementation of the mock authorization provider.
+     */
+    @DataProvider(name = "roles")
+    public Object[][] encryptionProvider() {
+        return new Object[][]{
+                // Schema: role, originalRole, whether authorization should pass
+
+                // Client conditions where original role isn't passed or is blank
+                {"pass.client", null, Boolean.TRUE},
+                {"pass.client", " ", Boolean.TRUE},
+                {"fail.client", null, Boolean.FALSE},
+                {"fail.client", " ", Boolean.FALSE},
+
+                // Proxy conditions where original role isn't passed or is blank
+                {"pass.proxy", null, Boolean.FALSE},
+                {"pass.proxy", " ", Boolean.FALSE},
+                {"fail.proxy", null, Boolean.FALSE},
+                {"fail.proxy", " ", Boolean.FALSE},
+
+                // Normal proxy and client conditions
+                {"pass.proxy", "pass.client", Boolean.TRUE},
+                {"pass.proxy", "fail.client", Boolean.FALSE},
+                {"fail.proxy", "pass.client", Boolean.FALSE},
+                {"fail.proxy", "fail.client", Boolean.FALSE},
+
+                // Not proxy with original principal
+                {"pass.not-proxy", "pass.client", Boolean.FALSE}, // non proxy role can't pass original role
+                {"pass.not-proxy", "fail.client", Boolean.FALSE},
+                {"fail.not-proxy", "pass.client", Boolean.FALSE},
+                {"fail.not-proxy", "fail.client", Boolean.FALSE},
+
+                // Covers an unlikely scenario, but valid in the context of this test
+                {null, "pass.proxy", Boolean.FALSE},
+        };
+    }
+
+    private void checkResult(boolean expected, boolean actual) {
+        if (expected) {
+            assertTrue(actual);
+        } else {
+            assertFalse(actual);
+        }
+    }
+
+    @Test(dataProvider = "roles")
+    public void testAllowTenantOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowTenantOperationAsync("tenant",
+                TenantOperation.DELETE_NAMESPACE, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testNamespaceOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowNamespaceOperationAsync(NamespaceName.get("public/default"),
+                NamespaceOperation.PACKAGES, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testTopicOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowTopicOperationAsync(TopicName.get("topic"),
+                TopicOperation.PRODUCE, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testNamespacePolicyOperationAsync(String role, String originalRole, boolean shouldPass)
+            throws Exception {
+        boolean isAuthorized = authorizationService.allowNamespacePolicyOperationAsync(
+                NamespaceName.get("public/default"), PolicyName.ALL, PolicyOperation.READ, originalRole, role, null)
+                .get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testTopicPolicyOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowTopicPolicyOperationAsync(TopicName.get("topic"),
+                PolicyName.ALL, PolicyOperation.READ, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+}
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java
new file mode 100644
index 0000000000000..4c939cbd9723a
--- /dev/null
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authorization;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.NamespaceOperation;
+import org.apache.pulsar.common.policies.data.PolicyName;
+import org.apache.pulsar.common.policies.data.PolicyOperation;
+import org.apache.pulsar.common.policies.data.TenantOperation;
+import org.apache.pulsar.common.policies.data.TopicOperation;
+
+/**
+ * Mock implementation of the authorization provider interface used for testing.
+ * A role is authorized if it starts with "pass".
+ */
+public class MockAuthorizationProvider implements AuthorizationProvider {
+
+    private CompletableFuture shouldPass(String role) {
+        return CompletableFuture.completedFuture(role != null && role.startsWith("pass"));
+    }
+
+    @Override
+    public CompletableFuture canProduceAsync(TopicName topicName, String role,
+                                                      AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture canConsumeAsync(TopicName topicName, String role,
+                                                      AuthenticationDataSource authenticationData, String subscription) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture canLookupAsync(TopicName topicName, String role,
+                                                     AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role,
+                                                            AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role,
+                                                          AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowTenantOperationAsync(String tenantName, String role,
+                                                                TenantOperation operation,
+                                                                AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowNamespaceOperationAsync(NamespaceName namespaceName, String role,
+                                                                   NamespaceOperation operation,
+                                                                   AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceName namespaceName, PolicyName policy,
+                                                                         PolicyOperation operation, String role,
+                                                                         AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, TopicOperation operation,
+                                                               AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic, String role, PolicyName policy,
+                                                                     PolicyOperation operation,
+                                                                     AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+
+    @Override
+    public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role,
+                                                        AuthenticationDataSource authenticationData) {
+        return null;
+    }
+
+
+    @Override
+    public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role,
+                                                        String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
+                                                                    Set roles, String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
+                                                                     String role, String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role,
+                                                        String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+}

From ac574d58052fd6aef56175f06fe17017b40e9dc6 Mon Sep 17 00:00:00 2001
From: laminar 
Date: Tue, 21 Mar 2023 07:04:59 +0800
Subject: [PATCH 094/404] [improve][fn] Support processingGuarantees
 "EFFECTIVELY_ONCE" in python function  (#18929)

Signed-off-by: laminar 
---
 .../instance/src/main/python/contextimpl.py   | 13 ++++++
 .../src/main/python/python_instance.py        | 43 ++++++++++++++++---
 .../src/main/python/python_instance_main.py   | 13 +++---
 .../src/test/python/test_python_instance.py   | 12 +++---
 4 files changed, 64 insertions(+), 17 deletions(-)

diff --git a/pulsar-functions/instance/src/main/python/contextimpl.py b/pulsar-functions/instance/src/main/python/contextimpl.py
index 14247e2d647a5..bfe7b23927ba7 100755
--- a/pulsar-functions/instance/src/main/python/contextimpl.py
+++ b/pulsar-functions/instance/src/main/python/contextimpl.py
@@ -89,6 +89,19 @@ def get_message_properties(self):
   def get_current_message_topic_name(self):
     return self.message.topic_name()
 
+  def get_message_sequence_id(self):
+    if not self.get_message_id():
+      return None
+    ledger_id = self.get_message_id().ledger_id()
+    entry_id = self.get_message_id().entry_id()
+    offset = (ledger_id << 28) | entry_id
+    return offset
+
+  def get_message_partition_index(self):
+    if not self.get_message_id():
+      return None
+    return self.get_message_id().partition()
+
   def get_partition_key(self):
     return self.message.partition_key()
 
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index c679288789536..57edbf954b376 100755
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -106,6 +106,7 @@ def __init__(self,
     self.execution_thread = None
     self.atmost_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATMOST_ONCE')
     self.atleast_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATLEAST_ONCE')
+    self.effectively_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('EFFECTIVELY_ONCE')
     self.manual = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('MANUAL')
     self.auto_ack = self.instance_config.function_details.autoAck
     self.contextimpl = None
@@ -143,11 +144,15 @@ def run(self):
     if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value("FAILOVER"):
       mode = pulsar._pulsar.ConsumerType.Failover
 
+    if self.instance_config.function_details.retainOrdering or \
+      self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value("EFFECTIVELY_ONCE"):
+      mode = pulsar._pulsar.ConsumerType.Failover
+
     position = pulsar._pulsar.InitialPosition.Latest
     if self.instance_config.function_details.source.subscriptionPosition == Function_pb2.SubscriptionPosition.Value("EARLIEST"):
       position = pulsar._pulsar.InitialPosition.Earliest
 
-    subscription_name = self.instance_config.function_details.source.subscriptionName    
+    subscription_name = self.instance_config.function_details.source.subscriptionName
 
     if not (subscription_name and subscription_name.strip()):
       subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
@@ -293,7 +298,7 @@ def actual_execution(self):
 
   def done_producing(self, consumer, orig_message, topic, result, sent_message):
     if result == pulsar.Result.Ok:
-      if self.auto_ack and self.atleast_once:
+      if self.auto_ack and self.atleast_once or self.effectively_once:
         consumer.acknowledge(orig_message)
     else:
       error_msg = "Failed to publish to topic [%s] with error [%s] with src message id [%s]" % (topic, result, orig_message.message_id())
@@ -302,14 +307,27 @@ def done_producing(self, consumer, orig_message, topic, result, sent_message):
       # If producer fails send output then send neg ack for input message back to broker
       consumer.negative_acknowledge(orig_message)
 
-
   def process_result(self, output, msg):
-    if output is not None and self.instance_config.function_details.sink.topic != None and \
+    if output is not None and self.instance_config.function_details.sink.topic is not None and \
             len(self.instance_config.function_details.sink.topic) > 0:
       if self.output_serde is None:
         self.setup_output_serde()
+      if self.effectively_once:
+        if self.contextimpl.get_message_partition_index() is None or \
+                self.contextimpl.get_message_partition_index() >= 0:
+          Log.error("Partitioned topic is not available in effectively_once mode.")
+          raise Exception("Partitioned topic is not available in effectively_once mode.")
+
+        producer_id = self.instance_config.function_details.sink.topic
+        producer = self.contextimpl.publish_producers.get(producer_id)
+        if producer is None:
+          self.setup_producer(producer_name=producer_id)
+          self.contextimpl.publish_producers[producer_id] = self.producer
+          Log.info("Setup producer [%s] successfully in effectively_once mode." % self.producer.producer_name())
+
       if self.producer is None:
         self.setup_producer()
+        Log.info("Setup producer successfully.")
 
       # only serialize function output when output schema is not set
       output_object = output
@@ -318,9 +336,21 @@ def process_result(self, output, msg):
 
       if output_object is not None:
         props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())}
-        self.producer.send_async(output_object, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props)
+        if self.effectively_once:
+          self.producer.send_async(output_object,
+                                   partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()),
+                                   properties=props,
+                                   sequence_id=self.contextimpl.get_message_sequence_id())
+          Log.debug("Send message with sequence ID [%s] using the producer [%s] in effectively_once mode." %
+                    (self.contextimpl.get_message_sequence_id(), self.producer.producer_name()))
+        else:
+          self.producer.send_async(output_object,
+                                   partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()),
+                                   properties=props)
     elif self.auto_ack and self.atleast_once:
       msg.consumer.acknowledge(msg.message)
+    elif self.effectively_once:
+      msg.consumer.acknowledge_cumulative(msg.message)
 
   def setup_output_serde(self):
     if self.instance_config.function_details.sink.serDeClassName != None and \
@@ -332,7 +362,7 @@ def setup_output_serde(self):
       serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
       self.output_serde = serde_kclass()
 
-  def setup_producer(self):
+  def setup_producer(self, producer_name=None):
     if self.instance_config.function_details.sink.topic != None and \
             len(self.instance_config.function_details.sink.topic) > 0:
       Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic)
@@ -366,6 +396,7 @@ def setup_producer(self):
       self.producer = self.pulsar_client.create_producer(
         str(self.instance_config.function_details.sink.topic),
         schema=self.output_schema,
+        producer_name=producer_name,
         block_if_queue_full=True,
         batching_enabled=True,
         batching_type=batch_type,
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py
index 2d6520b2e99e9..9a923c7e3a18b 100755
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -164,11 +164,7 @@ def main():
     args.function_details = args.function_details[:-1]
   json_format.Parse(args.function_details, function_details)
 
-  if function_details.processingGuarantees == "EFFECTIVELY_ONCE":
-    print("Python instance current not support EFFECTIVELY_ONCE processing guarantees.")
-    sys.exit(1)
-
-  if function_details.autoAck == False and function_details.processingGuarantees == "ATMOST_ONCE" \
+  if function_details.autoAck is False and function_details.processingGuarantees == "ATMOST_ONCE" \
           or function_details.processingGuarantees == "ATLEAST_ONCE":
     print("When Guarantees == " + function_details.processingGuarantees + ", autoAck must be equal to true, "
           "This is a contradictory configuration, autoAck will be removed later,"
@@ -176,6 +172,13 @@ def main():
           "This is a contradictory configuration, autoAck will be removed later," 
           "Please refer to PIP: https://github.com/apache/pulsar/issues/15560")
     sys.exit(1)
+  if function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('EFFECTIVELY_ONCE'):
+    if len(function_details.source.inputSpecs.keys()) != 1 or function_details.sink.topic == "":
+      print("When Guarantees == EFFECTIVELY_ONCE you need to ensure that the following pre-requisites have been met:"
+            "1. deduplication is enabled"
+            "2. set ProcessingGuarantees to EFFECTIVELY_ONCE"
+            "3. the function has only one source topic and one sink topic (both are non-partitioned)")
+      sys.exit(1)
   if os.path.splitext(str(args.py))[1] == '.whl':
     if args.install_usercode_dependencies:
       cmd = "pip install -t %s" % os.path.dirname(os.path.abspath(str(args.py)))
diff --git a/pulsar-functions/instance/src/test/python/test_python_instance.py b/pulsar-functions/instance/src/test/python/test_python_instance.py
index bbd6ca1fe42f1..975ddfd90288d 100644
--- a/pulsar-functions/instance/src/test/python/test_python_instance.py
+++ b/pulsar-functions/instance/src/test/python/test_python_instance.py
@@ -25,7 +25,7 @@
 sys.modules['prometheus_client'] = Mock()
 
 from contextimpl import ContextImpl
-from python_instance import InstanceConfig
+from python_instance import PythonInstance, InstanceConfig
 from pulsar import Message
 
 import Function_pb2
@@ -49,14 +49,14 @@ def test_context_publish(self):
     function_id = 'test_function_id'
     function_version = 'test_function_version'
     function_details = Function_pb2.FunctionDetails()
-    max_buffered_tuples = 100;
+    max_buffered_tuples = 100
     instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples)
     logger = log.Log
     pulsar_client = Mock()
     producer = Mock()
     producer.send_async = Mock(return_value=None)
     pulsar_client.create_producer = Mock(return_value=producer)
-    user_code=__file__
+    user_code = __file__
     consumers = None
     context_impl = ContextImpl(instance_config, logger, pulsar_client, user_code, consumers, None, None, None, None)
 
@@ -77,11 +77,11 @@ def test_context_ack_partitionedtopic(self):
     function_id = 'test_function_id'
     function_version = 'test_function_version'
     function_details = Function_pb2.FunctionDetails()
-    max_buffered_tuples = 100;
+    max_buffered_tuples = 100
     instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples)
     logger = log.Log
     pulsar_client = Mock()
-    user_code=__file__
+    user_code = __file__
     consumer = Mock()
     consumer.acknowledge = Mock(return_value=None)
     consumers = {"mytopic" : consumer}
@@ -89,4 +89,4 @@ def test_context_ack_partitionedtopic(self):
     context_impl.ack("test_message_id", "mytopic-partition-3")
 
     args, kwargs = consumer.acknowledge.call_args
-    self.assertEqual(args[0], "test_message_id")
\ No newline at end of file
+    self.assertEqual(args[0], "test_message_id")

From 03f8b806ec177cdf6a5a82193a3fc82557000989 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Tue, 21 Mar 2023 03:14:33 +0100
Subject: [PATCH 095/404] [fix][io] KCA sink: handle null values with
 KeyValue schema (#19861)

Co-authored-by: Andrey Yegorov 
---
 .../io/kafka/connect/KafkaConnectSink.java    |  4 +-
 .../connect/schema/KafkaConnectData.java      |  9 +++
 .../schema/PulsarSchemaToKafkaSchema.java     | 80 ++++++++++++++++++-
 .../kafka/connect/KafkaConnectSinkTest.java   | 54 +++++++++++++
 4 files changed, 144 insertions(+), 3 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index efbad2ef47ae0..06f66f60380d9 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -437,8 +437,8 @@ protected SinkRecord toSinkRecord(Record sourceRecord) {
 
             if (nativeObject instanceof KeyValue) {
                 KeyValue kv = (KeyValue) nativeObject;
-                key = KafkaConnectData.getKafkaConnectData(kv.getKey(), keySchema);
-                value = KafkaConnectData.getKafkaConnectData(kv.getValue(), valueSchema);
+                key = KafkaConnectData.getKafkaConnectDataFromSchema(kv.getKey(), keySchema);
+                value = KafkaConnectData.getKafkaConnectDataFromSchema(kv.getValue(), valueSchema);
             } else if (nativeObject != null) {
                 throw new IllegalStateException("Cannot extract KeyValue data from " + nativeObject.getClass());
             } else {
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java
index 757241d411034..a308ef01ddcf1 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java
@@ -54,6 +54,14 @@ private static List arrayToList(Object nativeObject, Schema kafkaValueSc
         return out;
     }
 
+
+    public static Object getKafkaConnectDataFromSchema(Object nativeObject, Schema kafkaSchema) {
+        if (kafkaSchema != null && nativeObject == null) {
+            return null;
+        }
+        return getKafkaConnectData(nativeObject, kafkaSchema);
+    }
+
     @SuppressWarnings("unchecked")
     public static Object getKafkaConnectData(Object nativeObject, Schema kafkaSchema) {
         if (kafkaSchema == null) {
@@ -380,6 +388,7 @@ private static Object defaultOrThrow(Schema kafkaSchema) {
         if (kafkaSchema.isOptional()) {
             return null;
         }
+
         throw new DataException("Invalid null value for required " + kafkaSchema.type() + " field");
     }
 }
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
index 2eb6573374cdb..faf28585e8aed 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
@@ -26,11 +26,14 @@
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import io.confluent.connect.avro.AvroData;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.connect.data.Date;
 import org.apache.kafka.connect.data.Decimal;
+import org.apache.kafka.connect.data.Field;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Time;
@@ -41,6 +44,76 @@
 
 @Slf4j
 public class PulsarSchemaToKafkaSchema {
+
+    private static class OptionalForcingSchema implements Schema {
+
+        Schema sourceSchema;
+
+        public OptionalForcingSchema(Schema sourceSchema) {
+            this.sourceSchema = sourceSchema;
+        }
+
+        @Override
+        public Type type() {
+            return sourceSchema.type();
+        }
+
+        @Override
+        public boolean isOptional() {
+            return true;
+        }
+
+        @Override
+        public Object defaultValue() {
+            return sourceSchema.defaultValue();
+        }
+
+        @Override
+        public String name() {
+            return sourceSchema.name();
+        }
+
+        @Override
+        public Integer version() {
+            return sourceSchema.version();
+        }
+
+        @Override
+        public String doc() {
+            return sourceSchema.doc();
+        }
+
+        @Override
+        public Map parameters() {
+            return sourceSchema.parameters();
+        }
+
+        @Override
+        public Schema keySchema() {
+            return sourceSchema.keySchema();
+        }
+
+        @Override
+        public Schema valueSchema() {
+            return sourceSchema.valueSchema();
+        }
+
+        @Override
+        public List fields() {
+            return sourceSchema.fields();
+        }
+
+        @Override
+        public Field field(String s) {
+            return sourceSchema.field(s);
+        }
+
+        @Override
+        public Schema schema() {
+            return sourceSchema.schema();
+        }
+    }
+
     private static final ImmutableMap pulsarSchemaTypeToKafkaSchema;
     private static final ImmutableSet kafkaLogicalSchemas;
     private static final AvroData avroData = new AvroData(1000);
@@ -80,6 +153,11 @@ private static org.apache.avro.Schema parseAvroSchema(String schemaJson) {
         return parser.parse(schemaJson);
     }
 
+    public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
+        Schema s = getKafkaConnectSchema(pulsarSchema);
+        return new OptionalForcingSchema(s);
+    }
+
     public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
         if (pulsarSchema == null || pulsarSchema.getSchemaInfo() == null) {
             throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null);
@@ -122,7 +200,7 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p
                 if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
                     KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
                     return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()),
-                                             getKafkaConnectSchema(kvSchema.getValueSchema()))
+                                    getOptionalKafkaConnectSchema(kvSchema.getValueSchema()))
                                 .build();
                 }
                 org.apache.avro.Schema avroSchema = parseAvroSchema(
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index 567562d338b98..e9d454ed2fd5a 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -51,6 +51,9 @@
 import org.apache.pulsar.client.api.schema.Field;
 import org.apache.pulsar.client.api.schema.GenericObject;
 import org.apache.pulsar.client.api.schema.GenericRecord;
+import org.apache.pulsar.client.api.schema.GenericSchema;
+import org.apache.pulsar.client.api.schema.RecordSchemaBuilder;
+import org.apache.pulsar.client.api.schema.SchemaBuilder;
 import org.apache.pulsar.client.api.schema.SchemaDefinition;
 import org.apache.pulsar.client.impl.BatchMessageIdImpl;
 import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -60,6 +63,7 @@
 import org.apache.pulsar.client.impl.schema.JSONSchema;
 import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
 import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord;
+import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
 import org.apache.pulsar.client.util.MessageIdUtils;
 import org.apache.pulsar.common.schema.KeyValue;
 import org.apache.pulsar.common.schema.SchemaInfo;
@@ -734,6 +738,56 @@ public void schemaKeyValueSchemaTest() throws Exception {
         Assert.assertEquals(key, 11);
     }
 
+    @Test
+    public void schemaKeyValueSchemaNullValueTest() throws Exception {
+        RecordSchemaBuilder builder = SchemaBuilder
+                .record("test");
+        builder.field("test").type(SchemaType.STRING);
+        GenericSchema schema = GenericAvroSchema.of(builder.build(SchemaType.AVRO));
+        KeyValue kv = new KeyValue<>(11, null);
+        SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, schema), 11,
+                "INT32", null, "STRUCT");
+        Assert.assertNull(sinkRecord.value());
+        int key = (int) sinkRecord.key();
+        Assert.assertEquals(key, 11);
+    }
+
+    @Test
+    public void schemaKeyValueSchemaNullValueNoUnwrapTest() throws Exception {
+        props.put("unwrapKeyValueIfAvailable", "false");
+        JSONSchema jsonSchema = JSONSchema
+                .of(SchemaDefinition.builder()
+                        .withPojo(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class)
+                        .withAlwaysAllowNull(true)
+                        .build());
+        KeyValue kv = new KeyValue<>(11, null);
+        Map expected = new HashMap();
+        expected.put("11", null);
+        SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, jsonSchema), "key",
+                "STRING", expected, "MAP");
+        Assert.assertNull(((Map)sinkRecord.value()).get(11));
+        String key =(String)sinkRecord.key();
+        Assert.assertEquals(key, "key");
+    }
+
+    @Test
+    public void schemaKeyValueSchemaNullValueNoUnwrapTestAvro() throws Exception {
+        props.put("unwrapKeyValueIfAvailable", "false");
+        RecordSchemaBuilder builder = SchemaBuilder
+                .record("test");
+        builder.property("op", "test");
+        builder.field("test").type(SchemaType.STRING);
+        GenericSchema schema = GenericAvroSchema.of(builder.build(SchemaType.AVRO));
+        KeyValue kv = new KeyValue<>(11, null);
+        Map expected = new HashMap();
+        expected.put("11", null);
+        SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, schema), "key",
+                "STRING", expected, "MAP");
+        Assert.assertNull(((Map)sinkRecord.value()).get(11));
+        String key =(String)sinkRecord.key();
+        Assert.assertEquals(key, "key");
+    }
+
     @Test
     public void kafkaLogicalTypesTimestampTest() {
         Schema schema = new TestSchema(SchemaInfoImpl.builder()

From 1e44ba179bfc189d22dff046a1887e12842b0851 Mon Sep 17 00:00:00 2001
From: Zike Yang 
Date: Tue, 21 Mar 2023 16:17:41 +0800
Subject: [PATCH 096/404] [fix][doc] The `messageIds` in `onNegativeAcksSend`
 shouldn't be null (#19852)

### Motivation

The `onNegativeAcksSend` is only called by:
https://github.com/apache/pulsar/blob/80c5791b87482bee3392308ecef45f455f8de885/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java#L83

It will not be null. The statement `null if acknowledge fail.` is incorrect.

### Modifications

* Remove the incorrect statement and refine the doc.

Signed-off-by: Zike Yang 
Co-authored-by: Jun Ma <60642177+momo-jun@users.noreply.github.com>
---
 .../java/org/apache/pulsar/client/api/ConsumerInterceptor.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java
index d6d91cd88500b..be2f9b0f10826 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java
@@ -107,7 +107,7 @@ public interface ConsumerInterceptor extends AutoCloseable {
      * 
Any exception thrown by this method will be ignored by the caller.
      *
      * @param consumer the consumer which contains the interceptor
-     * @param messageIds message to ack, null if acknowledge fail.
+     * @param messageIds the set of message ids to negatively ack
      */
     void onNegativeAcksSend(Consumer consumer, Set messageIds);
 

From 905e8ef730c96cbb43ad5c892f1dbfc3cd1521cc Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Tue, 21 Mar 2023 21:31:22 +0800
Subject: [PATCH 097/404] [improve][meta] Make session notification to be
 async. (#19869)

---
 .../apache/pulsar/common/util/FutureUtil.java | 26 +++++++++++++++++++
 .../coordination/impl/LeaderElectionImpl.java | 23 ++++++++--------
 .../coordination/impl/LockManagerImpl.java    | 22 +++++++---------
 3 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
index 531769a1e452b..2b082b4a7899b 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -36,6 +37,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import javax.annotation.concurrent.ThreadSafe;
 
 /**
@@ -238,6 +240,30 @@ public static  CompletableFuture addTimeoutHandling(CompletableFuture f
         return future;
     }
 
+    /**
+     * @throws RejectedExecutionException if this task cannot be accepted for execution
+     * @throws NullPointerException if one of params is null
+     */
+    public static  @Nonnull CompletableFuture composeAsync(Supplier> futureSupplier,
+                                                                 Executor executor) {
+        Objects.requireNonNull(futureSupplier);
+        Objects.requireNonNull(executor);
+        final CompletableFuture future = new CompletableFuture<>();
+        try {
+            executor.execute(() -> futureSupplier.get().whenComplete((result, error) -> {
+                if (error != null) {
+                    future.completeExceptionally(error);
+                    return;
+                }
+                future.complete(result);
+            }));
+        } catch (RejectedExecutionException ex) {
+            future.completeExceptionally(ex);
+        }
+        return future;
+    }
+
+
     /**
      * Creates a low-overhead timeout exception which is performance optimized to minimize allocations
      * and cpu consumption. It sets the stacktrace of the exception to the given source class and
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index c1121b1309c2c..11ae62226e7cd 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -24,7 +24,6 @@
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -32,6 +31,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.common.concurrent.FutureUtils;
 import org.apache.bookkeeper.common.util.SafeRunnable;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.GetResult;
 import org.apache.pulsar.metadata.api.MetadataCache;
 import org.apache.pulsar.metadata.api.MetadataCacheConfig;
@@ -62,6 +62,7 @@ class LeaderElectionImpl implements LeaderElection {
     private Optional proposedValue;
 
     private final ScheduledExecutorService executor;
+    private final FutureUtil.Sequencer sequencer;
 
     private enum InternalState {
         Init, ElectionInProgress, LeaderIsPresent, Closed
@@ -85,7 +86,7 @@ private enum InternalState {
         this.internalState = InternalState.Init;
         this.stateChangesListener = stateChangesListener;
         this.executor = executor;
-
+        this.sequencer = FutureUtil.Sequencer.create();
         store.registerListener(this::handlePathNotification);
         store.registerSessionListener(this::handleSessionNotification);
         updateCachedValueFuture = executor.scheduleWithFixedDelay(SafeRunnable.safeRun(this::getLeaderValue),
@@ -277,18 +278,18 @@ public Optional getLeaderValueIfPresent() {
 
     private void handleSessionNotification(SessionEvent event) {
         // Ensure we're only processing one session event at a time.
-        executor.execute(SafeRunnable.safeRun(() -> {
+        sequencer.sequential(() -> FutureUtil.composeAsync(() -> {
             if (event == SessionEvent.SessionReestablished) {
                 log.info("Revalidating leadership for {}", path);
-
-                try {
-                    LeaderElectionState les = elect().get();
-                    log.info("Resynced leadership for {} - State: {}", path, les);
-                } catch (ExecutionException | InterruptedException e) {
-                    log.warn("Failure when processing session event", e);
-                }
+                return elect().thenAccept(leaderState -> {
+                    log.info("Resynced leadership for {} - State: {}", path, leaderState);
+                }).exceptionally(ex -> {
+                    log.warn("Failure when processing session event", ex);
+                    return null;
+                });
             }
-        }));
+            return CompletableFuture.completedFuture(null);
+        }, executor));
     }
 
     private void handlePathNotification(Notification notification) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
index f5e7e0528bd7b..4da6b7998a0c4 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
@@ -27,11 +27,9 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.bookkeeper.common.util.SafeRunnable;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataCache;
 import org.apache.pulsar.metadata.api.MetadataSerde;
@@ -53,6 +51,7 @@ class LockManagerImpl implements LockManager {
     private final MetadataStoreExtended store;
     private final MetadataCache cache;
     private final MetadataSerde serde;
+    private final FutureUtil.Sequencer sequencer;
     private final ExecutorService executor;
 
     private enum State {
@@ -72,6 +71,7 @@ private enum State {
         this.cache = store.getMetadataCache(serde);
         this.serde = serde;
         this.executor = executor;
+        this.sequencer = FutureUtil.Sequencer.create();
         store.registerSessionListener(this::handleSessionEvent);
         store.registerListener(this::handleDataNotification);
     }
@@ -118,9 +118,8 @@ public CompletableFuture> acquireLock(String path, T value) {
     private void handleSessionEvent(SessionEvent se) {
         // We want to make sure we're processing one event at a time and that we're done with one event before going
         // for the next one.
-        executor.execute(SafeRunnable.safeRun(() -> {
-            List> futures = new ArrayList<>();
-
+        sequencer.sequential(() -> FutureUtil.composeAsync(() -> {
+            final List> futures = new ArrayList<>();
             if (se == SessionEvent.SessionReestablished) {
                 log.info("Metadata store session has been re-established. Revalidating all the existing locks.");
                 for (ResourceLockImpl lock : locks.values()) {
@@ -133,13 +132,12 @@ private void handleSessionEvent(SessionEvent se) {
                     futures.add(lock.revalidateIfNeededAfterReconnection());
                 }
             }
-
-            try {
-                FutureUtil.waitForAll(futures).get();
-            } catch (ExecutionException | InterruptedException e) {
-                log.warn("Failure when processing session event", e);
-            }
-        }));
+            return FutureUtil.waitForAll(futures)
+                    .exceptionally(ex -> {
+                        log.warn("Failure when processing session event", ex);
+                        return null;
+                    });
+        }, executor));
     }
 
     private void handleDataNotification(Notification n) {

From a50ecda11f947e5120cfb01adbfd46f22b76f1f2 Mon Sep 17 00:00:00 2001
From: Nihar Rathod 
Date: Tue, 21 Mar 2023 19:30:54 +0530
Subject: [PATCH 098/404] [fix][test]
 PulsarStandaloneTest.testMetadataInitialization (#19842)

---
 .../test/resources/configurations/standalone_no_client_auth.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf
index 573d6cdba9933..d9411e655ad5b 100644
--- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf
+++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf
@@ -29,3 +29,4 @@ authenticationEnabled=true
 authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider
 brokerClientAuthenticationPlugin=
 brokerClientAuthenticationParameters=
+loadBalancerOverrideBrokerNicSpeedGbps=2
\ No newline at end of file

From 30d2469086fea989ac8baf059df8e69c66a68d89 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 21 Mar 2023 07:33:00 -0700
Subject: [PATCH 099/404] [improve][broker] PIP-192 Excluded bundles with
 isolation policy or anti-affinity-group policy from topk load bundles
 (#19742)

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

Bundles with isolation policies or anti-affinity-group policies are not ideal targets to auto-unload as destination brokers are limited.

### Modifications
This PR
- Excluded bundles with isolation policy or anti-affinity-group policy from topk load bundles
- Introduced a config `loadBalancerSheddingBundlesWithPoliciesEnabled ` to control this behavior.
---
 .../pulsar/broker/ServiceConfiguration.java   |  10 +
 .../extensions/models/TopKBundles.java        |  79 ++++++--
 .../AntiAffinityGroupPolicyHelper.java        |  31 +--
 .../reporter/TopBundleLoadDataReporter.java   |   2 +-
 .../extensions/scheduler/TransferShedder.java |   7 +
 ...tiAffinityNamespaceGroupExtensionTest.java |  30 +++
 .../extensions/models/TopKBundlesTest.java    | 182 ++++++++++++++++--
 .../TopBundleLoadDataReporterTest.java        |  39 +++-
 .../scheduler/TransferShedderTest.java        |  42 ++--
 9 files changed, 356 insertions(+), 66 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ff242888ae0b6..ba82667690dc5 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2600,6 +2600,16 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     )
     private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800;
 
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Option to automatically unload namespace bundles with affinity(isolation) "
+                    + "or anti-affinity group policies."
+                    + "Such bundles are not ideal targets to auto-unload as destination brokers are limited."
+                    + "(only used in load balancer extension logics)"
+    )
+    private boolean loadBalancerSheddingBundlesWithPoliciesEnabled = false;
+
     /**** --- Replication. --- ****/
     @FieldContext(
         category = CATEGORY_REPLICATION,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
index c189005b9539c..d49361741bec4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
@@ -24,10 +24,14 @@
 import java.util.Map;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.NoArgsConstructor;
 import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 
 /**
@@ -36,7 +40,7 @@
 @Getter
 @ToString
 @EqualsAndHashCode
-@NoArgsConstructor
+@Slf4j
 public class TopKBundles {
 
     // temp array for sorting
@@ -44,6 +48,15 @@ public class TopKBundles {
 
     private final TopBundlesLoadData loadData = new TopBundlesLoadData();
 
+    private final PulsarService pulsar;
+
+    private final SimpleResourceAllocationPolicies allocationPolicies;
+
+    public TopKBundles(PulsarService pulsar) {
+        this.pulsar = pulsar;
+        this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
+    }
+
     /**
      * Update the topK bundles from the input bundleStats.
      *
@@ -52,26 +65,35 @@ public class TopKBundles {
      */
     public void update(Map bundleStats, int topk) {
         arr.clear();
-        for (var etr : bundleStats.entrySet()) {
-            if (etr.getKey().startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) {
-                continue;
+        try {
+            var isLoadBalancerSheddingBundlesWithPoliciesEnabled =
+                    pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled();
+            for (var etr : bundleStats.entrySet()) {
+                String bundle = etr.getKey();
+                if (bundle.startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) {
+                    continue;
+                }
+                if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled && hasPolicies(bundle)) {
+                    continue;
+                }
+                arr.add(etr);
             }
-            arr.add(etr);
-        }
-        var topKBundlesLoadData = loadData.getTopBundlesLoadData();
-        topKBundlesLoadData.clear();
-        if (arr.isEmpty()) {
-            return;
-        }
-        topk = Math.min(topk, arr.size());
-        partitionSort(arr, topk);
+            var topKBundlesLoadData = loadData.getTopBundlesLoadData();
+            topKBundlesLoadData.clear();
+            if (arr.isEmpty()) {
+                return;
+            }
+            topk = Math.min(topk, arr.size());
+            partitionSort(arr, topk);
 
-        for (int i = 0; i < topk; i++) {
-            var etr = arr.get(i);
-            topKBundlesLoadData.add(
-                    new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue()));
+            for (int i = 0; i < topk; i++) {
+                var etr = arr.get(i);
+                topKBundlesLoadData.add(
+                        new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue()));
+            }
+        } finally {
+            arr.clear();
         }
-        arr.clear();
     }
 
     static void partitionSort(List> arr, int k) {
@@ -109,4 +131,23 @@ static void partitionSort(List> arr, int
         }
         Collections.sort(arr.subList(0, end), (a, b) -> b.getValue().compareTo(a.getValue()));
     }
+
+    private boolean hasPolicies(String bundle) {
+        NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle));
+        if (allocationPolicies.areIsolationPoliciesPresent(namespace)) {
+            return true;
+        }
+
+        try {
+            var antiAffinityGroupOptional =
+                    LoadManagerShared.getNamespaceAntiAffinityGroup(pulsar, namespace.toString());
+            if (antiAffinityGroupOptional.isPresent()) {
+                return true;
+            }
+        } catch (MetadataStoreException e) {
+            log.error("Failed to get localPolicies for bundle:{}.", bundle, e);
+            throw new RuntimeException(e);
+        }
+        return false;
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
index c8332a1d7b5e6..69e3302bebd50 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -54,32 +54,37 @@ public boolean canUnload(
             String bundle,
             String srcBroker,
             Optional dstBroker) {
+
         try {
             var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
                     pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
-            if (antiAffinityGroupOptional.isPresent()) {
+            if (antiAffinityGroupOptional.isEmpty()) {
+                return true;
+            }
 
-                // copy to retain the input brokers
-                Map candidates = new HashMap<>(brokers);
+            // bundle has anti-affinityGroup
+            if (!pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) {
+                return false;
+            }
 
-                filter(candidates, bundle);
+            // copy to retain the input brokers
+            Map candidates = new HashMap<>(brokers);
 
-                candidates.remove(srcBroker);
+            filter(candidates, bundle);
 
-                // unload case
-                if (dstBroker.isEmpty()) {
-                    return !candidates.isEmpty();
-                }
+            candidates.remove(srcBroker);
 
-                // transfer case
-                return candidates.containsKey(dstBroker.get());
+            // unload case
+            if (dstBroker.isEmpty()) {
+                return !candidates.isEmpty();
             }
+
+            // transfer case
+            return candidates.containsKey(dstBroker.get());
         } catch (MetadataStoreException e) {
             log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e);
             return false;
         }
-
-        return true;
     }
 
     public void listenFailureDomainUpdate() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
index 59e328fc2be80..cc1a39add5d4e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
@@ -48,7 +48,7 @@ public TopBundleLoadDataReporter(PulsarService pulsar,
         this.lookupServiceAddress = lookupServiceAddress;
         this.bundleLoadDataStore = bundleLoadDataStore;
         this.lastBundleStatsUpdatedAt = 0;
-        this.topKBundles = new TopKBundles();
+        this.topKBundles = new TopKBundles(pulsar);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 3c67479bcc7cb..effddaf9c4159 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -456,6 +456,7 @@ private boolean isTransferable(LoadManagerContext context,
         if (pulsar == null || allocationPolicies == null) {
             return true;
         }
+
         String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
         final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
         NamespaceBundle namespaceBundle =
@@ -491,6 +492,12 @@ private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext cont
                 || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) {
             return true;
         }
+
+        // bundle has isolation policies.
+        if (!context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) {
+            return false;
+        }
+
         boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled();
         Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
index 32822c0f5b524..16b87195b1511 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -105,6 +105,8 @@ protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, Str
             brokersCopy.remove(srcBroker);
             var dstBroker = brokersCopy.entrySet().iterator().next().getKey();
 
+            // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = true
+            conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
             assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
                     "not-enabled-" + namespace + "/" + bundle,
                     srcBroker, Optional.of(dstBroker)));
@@ -128,6 +130,34 @@ protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, Str
             assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
                     namespaceBundle,
                     dstBroker, Optional.empty()));
+
+            // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = false
+            conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.empty()));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.of(srcBroker)));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.empty()));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.empty()));
+
+
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
index d759dd016955a..9b42163bd664b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
@@ -18,65 +18,221 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Random;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.broker.resources.LocalPoliciesResources;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
+import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
+import org.apache.pulsar.common.policies.data.LocalPolicies;
+import org.apache.pulsar.common.policies.data.NamespaceIsolationData;
+import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl;
+import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
 public class TopKBundlesTest {
+    private PulsarService pulsar;
+    private ServiceConfiguration configuration;
+    private NamespaceResources.IsolationPolicyResources isolationPolicyResources;
+    private PulsarResources pulsarResources;
+    private LocalPoliciesResources localPoliciesResources;
 
+    String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF";
+    String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF";
+    String bundle3 = "my-tenant/my-namespace3/0x00000000_0x0FFFFFFF";
+    String bundle4 = "my-tenant/my-namespace4/0x00000000_0x0FFFFFFF";
+
+    @BeforeMethod
+    public void init() throws MetadataStoreException {
+        pulsar = mock(PulsarService.class);
+        configuration = new ServiceConfiguration();
+        doReturn(configuration).when(pulsar).getConfiguration();
+        configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+        pulsarResources = mock(PulsarResources.class);
+        var namespaceResources = mock(NamespaceResources.class);
+
+        isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
+        doReturn(pulsarResources).when(pulsar).getPulsarResources();
+        doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
+        doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies();
+        doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any());
+
+        localPoliciesResources = mock(LocalPoliciesResources.class);
+        doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
+        doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any());
+
+    }
     @Test
     public void testTopBundlesLoadData() {
         Map bundleStats = new HashMap<>();
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
         stats1.msgRateIn = 500;
-        bundleStats.put("bundle-1", stats1);
+        bundleStats.put(bundle1, stats1);
 
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
         stats2.msgRateIn = 10000;
-        bundleStats.put("bundle-2", stats2);
+        bundleStats.put(bundle2, stats2);
 
         NamespaceBundleStats stats3 = new NamespaceBundleStats();
         stats3.msgRateIn = 100000;
-        bundleStats.put("bundle-3", stats3);
+        bundleStats.put(bundle3, stats3);
 
         NamespaceBundleStats stats4 = new NamespaceBundleStats();
         stats4.msgRateIn = 0;
-        bundleStats.put("bundle-4", stats4);
+        bundleStats.put(bundle4, stats4);
 
         topKBundles.update(bundleStats, 3);
         var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
         var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
         var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2);
 
-        assertEquals(top0.bundleName(), "bundle-3");
-        assertEquals(top1.bundleName(), "bundle-2");
-        assertEquals(top2.bundleName(), "bundle-1");
+        assertEquals(top0.bundleName(), bundle3);
+        assertEquals(top1.bundleName(), bundle2);
+        assertEquals(top2.bundleName(), bundle1);
     }
 
     @Test
     public void testSystemNamespace() {
         Map bundleStats = new HashMap<>();
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
         stats1.msgRateIn = 500;
-        bundleStats.put("pulsar/system/bundle-1", stats1);
+        bundleStats.put("pulsar/system/0x00000000_0x0FFFFFFF", stats1);
 
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
         stats2.msgRateIn = 10000;
-        bundleStats.put("pulsar/system/bundle-2", stats2);
+        bundleStats.put(bundle1, stats2);
 
         topKBundles.update(bundleStats, 2);
-        assertTrue(topKBundles.getLoadData().getTopBundlesLoadData().isEmpty());
+
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        assertEquals(top0.bundleName(), bundle1);
+    }
+
+
+    private  void setAntiAffinityGroup() throws MetadataStoreException {
+        LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup");
+        NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle2));
+        doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(eq(namespace));
+    }
+
+    private void setIsolationPolicy() throws MetadataStoreException {
+        Map parameters = new HashMap<>();
+        parameters.put("min_limit", "3");
+        parameters.put("usage_threshold", "90");
+        var policyData = Map.of("policy", (NamespaceIsolationDataImpl)
+                NamespaceIsolationData.builder()
+                        .namespaces(Collections.singletonList("my-tenant/my-namespace1.*"))
+                        .primary(Collections.singletonList("prod1-broker[1-3].messaging.use.example.com"))
+                        .secondary(Collections.singletonList("prod1-broker.*.use.example.com"))
+                        .autoFailoverPolicy(AutoFailoverPolicyData.builder()
+                                .policyType(AutoFailoverPolicyType.min_available)
+                                .parameters(parameters)
+                                .build()
+                        ).build());
+
+        NamespaceIsolationPolicies policies = new NamespaceIsolationPolicies(policyData);
+        doReturn(Optional.of(policies)).when(isolationPolicyResources).getIsolationDataPolicies(any());
+    }
+
+    @Test
+    public void testIsolationPolicy() throws MetadataStoreException {
+
+        setIsolationPolicy();
+
+        Map bundleStats = new HashMap<>();
+        var topKBundles = new TopKBundles(pulsar);
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.msgRateIn = 500;
+        bundleStats.put(bundle1, stats1);
+
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.msgRateIn = 10000;
+        bundleStats.put(bundle2, stats2);
+
+        topKBundles.update(bundleStats, 2);
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        assertEquals(top0.bundleName(), bundle2);
+    }
+
+
+    @Test
+    public void testAntiAffinityGroupPolicy() throws MetadataStoreException {
+
+        setAntiAffinityGroup();
+
+        Map bundleStats = new HashMap<>();
+        var topKBundles = new TopKBundles(pulsar);
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.msgRateIn = 500;
+        bundleStats.put(bundle1, stats1);
+
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.msgRateIn = 10000;
+        bundleStats.put(bundle2, stats2);
+
+        topKBundles.update(bundleStats, 2);
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        assertEquals(top0.bundleName(), bundle1);
+
+    }
+
+    @Test
+    public void testLoadBalancerSheddingBundlesWithPoliciesEnabledConfig() throws MetadataStoreException {
+
+        setIsolationPolicy();
+        setAntiAffinityGroup();
+
+        configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
+
+        Map bundleStats = new HashMap<>();
+        var topKBundles = new TopKBundles(pulsar);
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.msgRateIn = 500;
+        bundleStats.put(bundle1, stats1);
+
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.msgRateIn = 10000;
+        bundleStats.put(bundle2, stats2);
+
+        topKBundles.update(bundleStats, 2);
+
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 2);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
+
+        assertEquals(top0.bundleName(), bundle2);
+        assertEquals(top1.bundleName(), bundle1);
+
+        configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+
+        topKBundles.update(bundleStats, 2);
+
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 0);
     }
 
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
index ce2d3d8c3ea93..b5c415c405fbd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
@@ -27,6 +27,7 @@
 import static org.testng.Assert.assertNull;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
@@ -34,8 +35,12 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.resources.LocalPoliciesResources;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.PulsarStats;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -48,26 +53,44 @@ public class TopBundleLoadDataReporterTest {
     PulsarStats pulsarStats;
     Map bundleStats;
     ServiceConfiguration config;
+    private NamespaceResources.IsolationPolicyResources isolationPolicyResources;
+    private PulsarResources pulsarResources;
+    private LocalPoliciesResources localPoliciesResources;
+    String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF";
+    String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF";
 
     @BeforeMethod
-    void setup() {
+    void setup() throws MetadataStoreException {
         config = new ServiceConfiguration();
         pulsar = mock(PulsarService.class);
         store = mock(LoadDataStore.class);
         brokerService = mock(BrokerService.class);
         pulsarStats = mock(PulsarStats.class);
+        pulsarResources = mock(PulsarResources.class);
+        isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
+        var namespaceResources = mock(NamespaceResources.class);
+        localPoliciesResources = mock(LocalPoliciesResources.class);
+
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any());
 
+        doReturn(pulsarResources).when(pulsar).getPulsarResources();
+        doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
+        doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies();
+        doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any());
+
+        doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
+        doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any());
+
         bundleStats = new HashMap<>();
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
         stats1.msgRateIn = 500;
-        bundleStats.put("bundle-1", stats1);
+        bundleStats.put(bundle1, stats1);
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
         stats2.msgRateIn = 10000;
-        bundleStats.put("bundle-2", stats2);
+        bundleStats.put(bundle2, stats2);
         doReturn(bundleStats).when(brokerService).getBundleStats();
     }
 
@@ -81,25 +104,25 @@ public void testGenerateLoadData() throws IllegalAccessException {
         doReturn(1l).when(pulsarStats).getUpdatedAt();
         config.setLoadBalancerBundleLoadReportPercentage(100);
         var target = new TopBundleLoadDataReporter(pulsar, "", store);
-        var expected = new TopKBundles();
+        var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 2);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         config.setLoadBalancerBundleLoadReportPercentage(50);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
-        expected = new TopKBundles();
+        expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         config.setLoadBalancerBundleLoadReportPercentage(1);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
-        expected = new TopKBundles();
+        expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         doReturn(new HashMap()).when(brokerService).getBundleStats();
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
-        expected = new TopKBundles();
+        expected = new TopKBundles(pulsar);
         assertEquals(target.generateLoadData(), expected.getLoadData());
     }
 
@@ -116,7 +139,7 @@ public void testReportForce()  {
     public void testReport(){
         var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store);
         doReturn(1l).when(pulsarStats).getUpdatedAt();
-        var expected = new TopKBundles();
+        var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         target.reportAsync(false);
         verify(store, times(1)).pushAsync("broker-1", expected.getLoadData());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index a89f33bb98737..5ca9345d6def5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -97,8 +97,8 @@ public class TransferShedderTest {
     double setupLoadStd = 0.3982762860126121;
 
     PulsarService pulsar;
-    AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     ServiceConfiguration conf;
+    AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     LocalPoliciesResources localPoliciesResources;
     String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF";
     String bundleD2 = "my-tenant/my-namespaceD/0x0FFFFFFF_0xFFFFFFFF";
@@ -109,8 +109,7 @@ public class TransferShedderTest {
     public void init() throws MetadataStoreException {
         pulsar = mock(PulsarService.class);
         conf = new ServiceConfiguration();
-        doReturn(conf).when(pulsar).getConfiguration();
-
+        conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
         var pulsarResources = mock(PulsarResources.class);
         var namespaceResources = mock(NamespaceResources.class);
         var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
@@ -118,6 +117,7 @@ public void init() throws MetadataStoreException {
         var namespaceService = mock(NamespaceService.class);
         localPoliciesResources = mock(LocalPoliciesResources.class);
         antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
+        doReturn(conf).when(pulsar).getConfiguration();
         doReturn(namespaceService).when(pulsar).getNamespaceService();
         doReturn(pulsarResources).when(pulsar).getPulsarResources();
         doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
@@ -138,9 +138,9 @@ public void init() throws MetadataStoreException {
         }).when(factory).getBundle(anyString(), anyString());
         doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
     }
+
     public LoadManagerContext setupContext(){
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
 
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  2));
@@ -160,7 +160,6 @@ public LoadManagerContext setupContext(){
 
     public LoadManagerContext setupContext(int clusterSize) {
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
 
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
@@ -170,7 +169,7 @@ public LoadManagerContext setupContext(int clusterSize) {
             int brokerLoad = rand.nextInt(100);
             brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad));
             int bundleLoad = rand.nextInt(brokerLoad + 1);
-            topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("bundle" + i,
+            topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
                     bundleLoad, brokerLoad - bundleLoad));
         }
         return ctx;
@@ -199,7 +198,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
         namespaceBundleStats1.msgThroughputOut = load1;
         var namespaceBundleStats2 = new NamespaceBundleStats();
         namespaceBundleStats2.msgThroughputOut = load2;
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1,
                 bundlePrefix + "/0x0FFFFFFF_0xFFFFFFFF", namespaceBundleStats2), 2);
         return topKBundles.getLoadData();
@@ -208,7 +207,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
     public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) {
         var namespaceBundleStats1 = new NamespaceBundleStats();
         namespaceBundleStats1.msgThroughputOut = load1;
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1), 2);
         return topKBundles.getLoadData();
     }
@@ -220,7 +219,7 @@ public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace,
         namespaceBundleStats1.msgThroughputOut = load1 * 1e6;
         var namespaceBundleStats2 = new NamespaceBundleStats();
         namespaceBundleStats2.msgThroughputOut = load2 * 1e6;
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         topKBundles.update(Map.of(namespace + "/0x00000000_0x7FFFFFF", namespaceBundleStats1,
                 namespace + "/0x7FFFFFF_0xFFFFFFF", namespaceBundleStats2), 2);
         return topKBundles.getLoadData();
@@ -229,6 +228,8 @@ public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace,
     public LoadManagerContext getContext(){
         var ctx = mock(LoadManagerContext.class);
         var conf = new ServiceConfiguration();
+        conf.setLoadBalancerDebugModeEnabled(true);
+        conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
         var brokerLoadDataStore = new LoadDataStore() {
             Map map = new HashMap<>();
             @Override
@@ -461,8 +462,6 @@ public void testGetAvailableBrokersFailed() {
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
-        assertTrue(res.isEmpty());
         assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
@@ -501,6 +500,25 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
         assertEquals(res, expected);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
+
+
+        // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false;
+        doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
+        ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true);
+        ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+        res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+
+        // Test unload a has isolation policies broker.
+        ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false);
+        res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 2);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     public BrokerLookupData getLookupData() {
@@ -602,7 +620,7 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         var expected2 = new HashSet<>();
         expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
                 Success, Overloaded));
-        assertEquals(res, expected2);
+        assertEquals(res2, expected2);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }

From f294be37ce12e311a6bd87493cf0cba11fc6c2f8 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 22 Mar 2023 15:30:00 +0800
Subject: [PATCH 100/404] [fix] [ml] make the result of delete cursor is
 success if cursor is deleted (#19825)

When deleting the zk node of the cursor, if the exception `MetadataStoreException.NotFoundException`  occurs, the deletion is considered successful.
---
 .../mledger/impl/MetaStoreImpl.java           | 16 ++++++++----
 .../mledger/impl/ManagedLedgerTest.java       | 26 +++++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index bcb73553324dd..3bdaac1e8d80e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -47,6 +47,7 @@
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.compression.CompressionCodec;
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataStore;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.Notification;
@@ -292,7 +293,7 @@ public void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedC
     @Override
     public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCallback callback) {
         String path = PREFIX + ledgerName + "/" + cursorName;
-        log.info("[{}] Remove consumer={}", ledgerName, cursorName);
+        log.info("[{}] Remove cursor={}", ledgerName, cursorName);
 
         store.delete(path, Optional.empty())
                 .thenAcceptAsync(v -> {
@@ -301,11 +302,16 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal
                     }
                     callback.operationComplete(null, null);
                 }, executor.chooseThread(ledgerName))
-                .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                .exceptionallyAsync(ex -> {
+                    Throwable actEx = FutureUtil.unwrapCompletionException(ex);
+                    if (actEx instanceof MetadataStoreException.NotFoundException){
+                        log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName);
+                        callback.operationComplete(null, null);
+                        return null;
+                    }
+                    SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex)));
                     return null;
-                });
+                }, executor.chooseThread(ledgerName));
     }
 
     @Override
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index a4d8b75d00c96..dd30cde72e769 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -128,6 +128,7 @@
 import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
 import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.Stat;
 import org.apache.pulsar.metadata.api.extended.SessionEvent;
@@ -3969,4 +3970,29 @@ public void testGetEstimatedBacklogSize() throws Exception {
         Assert.assertEquals(ledger.getEstimatedBacklogSize(((PositionImpl) positions.get(9)).getNext()), 0);
         ledger.close();
     }
+
+    @Test
+    public void testDeleteCursorTwice() throws Exception {
+        ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("ml");
+        String cursorName = "cursor_1";
+        ml.openCursor(cursorName);
+        syncRemoveCursor(ml, cursorName);
+        syncRemoveCursor(ml, cursorName);
+    }
+
+    private void syncRemoveCursor(ManagedLedgerImpl ml, String cursorName){
+        CompletableFuture future = new CompletableFuture<>();
+        ml.getStore().asyncRemoveCursor(ml.name, cursorName, new MetaStoreCallback() {
+            @Override
+            public void operationComplete(Void result, Stat stat) {
+                future.complete(null);
+            }
+
+            @Override
+            public void operationFailed(MetaStoreException e) {
+                future.completeExceptionally(FutureUtil.unwrapCompletionException(e));
+            }
+        });
+        future.join();
+    }
 }

From a9037334a399af905fae94d2aefa5db339cbd5b1 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 22 Mar 2023 16:49:13 +0800
Subject: [PATCH 101/404] [fix] [admin] Make response code to 400 instead of
 500 when delete topic fails due to enabled geo-replication (#19879)

Motivation: As expected, If geo-replication is enabled, a topic cannot be deleted. However deleting that topic returns a 500, and no further info.
Modifications: Make response code to 400 instead of 500 when delete topic fails due to enabled geo-replication
---
 .../broker/admin/v2/PersistentTopics.java     |  4 ++-
 .../pulsar/broker/service/ReplicatorTest.java | 28 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index 7eeaec403d842..477cc8b57129d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -1106,7 +1106,9 @@ public void deleteTopic(
                         ex = new RestException(Response.Status.PRECONDITION_FAILED,
                                 t.getMessage());
                     }
-                    if (isManagedLedgerNotFoundException(t)) {
+                    if (t instanceof IllegalStateException){
+                        ex = new RestException(422/* Unprocessable entity*/, t.getMessage());
+                    } else if (isManagedLedgerNotFoundException(t)) {
                         ex = new RestException(Response.Status.NOT_FOUND,
                                 getTopicNotFoundErrorMessage(topicName.toString()));
                     } else if (!isRedirectException(ex)) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index ab4f6a5c7f8ba..4086c54a0ba4b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
@@ -869,6 +870,33 @@ public void testReplicatorProducerClosing() throws Exception {
         assertNull(producer);
     }
 
+    @Test
+    public void testDeleteTopicFailure() throws Exception {
+        final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
+        admin1.topics().createNonPartitionedTopic(topicName);
+        try {
+            admin1.topics().delete(topicName);
+            fail("Delete topic should fail if enabled replicator");
+        } catch (Exception ex) {
+            assertTrue(ex instanceof PulsarAdminException);
+            assertEquals(((PulsarAdminException) ex).getStatusCode(), 422/* Unprocessable entity*/);
+        }
+    }
+
+    @Test
+    public void testDeletePartitionedTopicFailure() throws Exception {
+        final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
+        admin1.topics().createPartitionedTopic(topicName, 2);
+        admin1.topics().createSubscription(topicName, "sub1", MessageId.earliest);
+        try {
+            admin1.topics().deletePartitionedTopic(topicName);
+            fail("Delete topic should fail if enabled replicator");
+        } catch (Exception ex) {
+            assertTrue(ex instanceof PulsarAdminException);
+            assertEquals(((PulsarAdminException) ex).getStatusCode(), 422/* Unprocessable entity*/);
+        }
+    }
+
     @Test(priority = 4, timeOut = 30000)
     public void testReplicatorProducerName() throws Exception {
         log.info("--- Starting ReplicatorTest::testReplicatorProducerName ---");

From a1fdbd294cab5c48dd2b092085465eb4075cc6a4 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Thu, 23 Mar 2023 20:09:57 +0800
Subject: [PATCH 102/404] [fix][broker] Fix NPE when update topic policy.
 (#19875)

---
 .../apache/pulsar/common/policies/data/TopicPolicies.java    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
index caa9cd3fd1daa..4a76170d116a3 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
@@ -20,6 +20,7 @@
 
 import com.google.common.collect.Sets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -188,4 +189,8 @@ public boolean isSchemaValidationEnforced() {
     public Set getReplicationClustersSet() {
         return replicationClusters != null ? Sets.newTreeSet(this.replicationClusters) : null;
     }
+
+    public Map getSubscriptionPolicies() {
+        return subscriptionPolicies == null ? Collections.emptyMap() : subscriptionPolicies;
+    }
 }

From c172a775fa0fc5762a9703669ed8ebcd2efbe042 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Fri, 24 Mar 2023 15:05:02 +0800
Subject: [PATCH 103/404] [fix][broker] Fix can't send ErrorCommand when
 message is null value (#19899)

---
 .../pulsar/client/api/ClientErrorsTest.java   | 29 +++++++++++++++++--
 .../pulsar/common/protocol/Commands.java      |  4 +--
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
index e8b9baa992c46..61c7a98602b69 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
@@ -22,16 +22,15 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import io.netty.channel.ChannelHandlerContext;
-
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-
 import lombok.Cleanup;
+import org.apache.bookkeeper.common.util.JsonUtil;
 import org.apache.pulsar.client.impl.ConsumerBase;
 import org.apache.pulsar.client.impl.PartitionedProducerImpl;
 import org.apache.pulsar.client.impl.ProducerBase;
@@ -831,4 +830,28 @@ public void testConsumerReconnect() throws Exception {
         mockBrokerService.resetHandleConnect();
         mockBrokerService.resetHandleSubscribe();
     }
+
+    @Test
+    public void testCommandErrorMessageIsNull() throws Exception {
+        @Cleanup
+        PulsarClient client = PulsarClient.builder().serviceUrl(mockBrokerService.getBrokerAddress()).build();
+
+        mockBrokerService.setHandleProducer((ctx, producer) -> {
+            try {
+                ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.AuthorizationError, null));
+            } catch (Exception e) {
+                fail("Send error command failed", e);
+            }
+        });
+
+        try {
+            client.newProducer().topic("persistent://prop/use/ns/t1").create();
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof PulsarClientException.AuthorizationException);
+            Map map = JsonUtil.fromJson(e.getMessage(), Map.class);
+            assertEquals(map.get("errorMsg"), "");
+        }
+        mockBrokerService.resetHandleProducer();
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
index 8a5684cf676b0..85c4d021fdf22 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
@@ -368,7 +368,7 @@ public static BaseCommand newErrorCommand(long requestId, ServerError serverErro
         cmd.setError()
                 .setRequestId(requestId)
                 .setError(serverError)
-                .setMessage(message);
+                .setMessage(message != null ? message : "");
         return cmd;
     }
 
@@ -401,7 +401,7 @@ public static BaseCommand newSendErrorCommand(long producerId, long sequenceId,
                 .setProducerId(producerId)
                 .setSequenceId(sequenceId)
                 .setError(error)
-                .setMessage(errorMsg);
+                .setMessage(errorMsg != null ? errorMsg : "");
         return cmd;
     }
 

From f1f8dab972b098be69ad35ab3d307f19284c4e48 Mon Sep 17 00:00:00 2001
From: Abhilash Mandaliya 
Date: Fri, 24 Mar 2023 16:51:22 +0530
Subject: [PATCH 104/404] [improve][io][broker] Updated
 org.reflections-reflections library version (#19898)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 257018b0b0598..3f33069b8c252 100644
--- a/pom.xml
+++ b/pom.xml
@@ -149,7 +149,7 @@ flexible messaging model and an intuitive client API.
     1.0.6
     1.0.2.3
     2.13.4.20221013
-    0.9.11
+    0.10.2
     1.6.2
     8.37
     0.40.2

From 1af0ff35109af066662324515e59d2b8bff8c899 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 24 Mar 2023 15:03:47 +0000
Subject: [PATCH 105/404] [improve][ci] Handle retry of maven downloads for
 more failure cases (#19903)

---
 .github/workflows/ci-go-functions.yaml           | 2 +-
 .github/workflows/ci-maven-cache-update.yaml     | 2 +-
 .github/workflows/ci-owasp-dependency-check.yaml | 2 +-
 .github/workflows/pulsar-ci-flaky.yaml           | 2 +-
 .github/workflows/pulsar-ci.yaml                 | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index f4d1ae0b99887..a34cb1a90908e 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -32,7 +32,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   preconditions:
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index 87570586fde6e..a5a88e2e0b8a6 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -42,7 +42,7 @@ on:
     - cron: '30 */12 * * *'
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   update-maven-dependencies-cache:
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index 194d88c582d42..dba1c92ca28dd 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,7 +24,7 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   run-owasp-dependency-check:
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index acfa66ff43c74..ea19bc3307819 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -36,7 +36,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 721a1d2eafc72..f7dbc755264d6 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -36,7 +36,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR

From 8081ee26d8d3727c720800a3453a798893763fee Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sat, 25 Mar 2023 15:39:37 +0800
Subject: [PATCH 106/404] [improve][broker][PIP-195] Make bucket merge
 operation asynchronous (#19873)

---
 .../bucket/BucketDelayedDeliveryTracker.java  | 139 +++++++++++-------
 .../delayed/bucket/ImmutableBucket.java       |  17 ++-
 .../broker/delayed/bucket/MutableBucket.java  |   6 +
 ...ayedMessageIndexBucketSnapshotFormat.proto |   1 +
 .../BookkeeperBucketSnapshotStorageTest.java  |   3 +
 .../BucketDelayedDeliveryTrackerTest.java     |  57 +++++--
 6 files changed, 154 insertions(+), 69 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 31fdaa6fb7667..1cf8bd2b20d17 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -90,6 +90,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final Table snapshotSegmentLastIndexTable;
 
+    private static final Long INVALID_BUCKET_ID = -1L;
+
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
@@ -246,12 +248,12 @@ private void afterCreateImmutableBucket(Pair immu
                     immutableBucket);
 
             immutableBucket.getSnapshotCreateFuture().ifPresent(createFuture -> {
-                CompletableFuture future = createFuture.whenComplete((__, ex) -> {
+                CompletableFuture future = createFuture.handle((bucketId, ex) -> {
                     if (ex == null) {
                         immutableBucket.setSnapshotSegments(null);
                         log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(),
                                 immutableBucket.bucketKey());
-                        return;
+                        return bucketId;
                     }
 
                     //TODO Record create snapshot failed
@@ -277,6 +279,7 @@ private void afterCreateImmutableBucket(Pair immu
                         snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(),
                                 lastDelayedIndex.getTimestamp());
                     }
+                    return INVALID_BUCKET_ID;
                 });
                 immutableBucket.setSnapshotCreateFuture(future);
             });
@@ -308,12 +311,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
             lastMutableBucket.resetLastMutableBucketRange();
 
             if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) {
-                try {
-                    asyncMergeBucketSnapshot().get(2 * AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    // Ignore exception to merge bucket on the next schedule.
-                    log.error("[{}] An exception occurs when merge bucket snapshot.", dispatcher.getName(), e);
-                }
+                asyncMergeBucketSnapshot();
             }
         }
 
@@ -341,18 +339,26 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
     private synchronized CompletableFuture asyncMergeBucketSnapshot() {
         List values = immutableBuckets.asMapOfRanges().values().stream().toList();
         long minNumberMessages = Long.MAX_VALUE;
+        long minScheduleTimestamp = Long.MAX_VALUE;
         int minIndex = -1;
         for (int i = 0; i + 1 < values.size(); i++) {
             ImmutableBucket bucketL = values.get(i);
             ImmutableBucket bucketR = values.get(i + 1);
-            long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages;
-            if (numberMessages < minNumberMessages) {
-                minNumberMessages = (int) numberMessages;
-                if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()
-                        && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId()
-                        && bucketL.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()
-                        && bucketR.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()) {
-                    minIndex = i;
+            // We should skip the bucket which last segment already been load to memory, avoid record replicated index.
+            if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()
+                    && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId()
+                    // Skip the bucket that is merging
+                    && !bucketL.merging && !bucketR.merging){
+                long scheduleTimestamp =
+                        Math.min(bucketL.firstScheduleTimestamps.get(bucketL.currentSegmentEntryId + 1),
+                                bucketR.firstScheduleTimestamps.get(bucketR.currentSegmentEntryId + 1));
+                long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages;
+                if (scheduleTimestamp <= minScheduleTimestamp) {
+                    minScheduleTimestamp = scheduleTimestamp;
+                    if (numberMessages < minNumberMessages) {
+                        minNumberMessages = numberMessages;
+                        minIndex = i;
+                    }
                 }
             }
         }
@@ -369,7 +375,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
             log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(),
                     immutableBucketA.bucketKey(), immutableBucketB.bucketKey());
         }
+
+        immutableBucketA.merging = true;
+        immutableBucketB.merging = true;
         return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> {
+            synchronized (this) {
+                immutableBucketA.merging = false;
+                immutableBucketB.merging = false;
+            }
             if (ex != null) {
                 log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}",
                         dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex);
@@ -382,46 +395,58 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
 
     private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA,
                                                                           ImmutableBucket bucketB) {
-        CompletableFuture> futureA =
-                bucketA.getRemainSnapshotSegment();
-        CompletableFuture> futureB =
-                bucketB.getRemainSnapshotSegment();
-        return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap)
-                .thenAccept(combinedDelayedIndexQueue -> {
-                    Pair immutableBucketDelayedIndexPair =
-                            lastMutableBucket.createImmutableBucketAndAsyncPersistent(
-                                    timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment,
-                                    sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
-                                    bucketB.endLedgerId);
-
-                    // Merge bit map to new bucket
-                    Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
-                    Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap();
-                    Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA);
-                    delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> {
-                        delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> {
-                            if (bitMapA == null) {
-                                return bitMapB;
-                            }
+        CompletableFuture createAFuture = bucketA.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
+        CompletableFuture createBFuture = bucketB.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
 
-                            bitMapA.or(bitMapB);
-                            return bitMapA;
-                        });
-                    });
-                    immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap);
-
-                    afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
+        return CompletableFuture.allOf(createAFuture, createBFuture).thenCompose(bucketId -> {
+            if (INVALID_BUCKET_ID.equals(createAFuture.join()) || INVALID_BUCKET_ID.equals(createBFuture.join())) {
+                return FutureUtil.failedFuture(new RuntimeException("Can't merge buckets due to bucket create failed"));
+            }
 
-                    immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture()
-                            .orElse(NULL_LONG_PROMISE).thenCompose(___ -> {
-                        CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot();
-                        CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot();
-                        return CompletableFuture.allOf(removeAFuture, removeBFuture);
+            CompletableFuture> futureA =
+                    bucketA.getRemainSnapshotSegment();
+            CompletableFuture> futureB =
+                    bucketB.getRemainSnapshotSegment();
+            return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap)
+                    .thenAccept(combinedDelayedIndexQueue -> {
+                        synchronized (BucketDelayedDeliveryTracker.this) {
+                            Pair immutableBucketDelayedIndexPair =
+                                    lastMutableBucket.createImmutableBucketAndAsyncPersistent(
+                                            timeStepPerBucketSnapshotSegmentInMillis,
+                                            maxIndexesPerBucketSnapshotSegment,
+                                            sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
+                                            bucketB.endLedgerId);
+
+                            // Merge bit map to new bucket
+                            Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
+                            Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap();
+                            Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA);
+                            delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> {
+                                delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> {
+                                    if (bitMapA == null) {
+                                        return bitMapB;
+                                    }
+
+                                    bitMapA.or(bitMapB);
+                                    return bitMapA;
+                                });
+                            });
+                            immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap);
+
+                            afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
+
+                            immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture()
+                                    .orElse(NULL_LONG_PROMISE).thenCompose(___ -> {
+                                        CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot();
+                                        CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot();
+                                        return CompletableFuture.allOf(removeAFuture, removeBFuture);
+                                    });
+
+                            immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
+                            immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
+                        }
                     });
-
-                    immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
-                    immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
-                });
+        });
     }
 
     @Override
@@ -477,6 +502,12 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
             ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId);
             if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) {
+                if (bucket.merging) {
+                    log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}",
+                            dispatcher.getName(), bucket.bucketKey());
+                    break;
+                }
+
                 final int preSegmentEntryId = bucket.currentSegmentEntryId;
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}",
@@ -525,7 +556,6 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                                     dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId);
                         }
                     }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS);
-                    snapshotSegmentLastIndexTable.remove(ledgerId, entryId);
                 } catch (Exception e) {
                     // Ignore exception to reload this segment on the next schedule.
                     log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}",
@@ -533,6 +563,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                     break;
                 }
             }
+            snapshotSegmentLastIndexTable.remove(ledgerId, entryId);
 
             positions.add(new PositionImpl(ledgerId, entryId));
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index c9223efa09243..ab1c285011d20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -22,6 +22,7 @@
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds;
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE;
 import com.google.protobuf.ByteString;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -44,7 +45,12 @@
 class ImmutableBucket extends Bucket {
 
     @Setter
-    private volatile List snapshotSegments;
+    private List snapshotSegments;
+
+    boolean merging = false;
+
+    @Setter
+    List firstScheduleTimestamps = new ArrayList<>();
 
     ImmutableBucket(String dispatcherName, ManagedCursor cursor,
                     BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) {
@@ -92,6 +98,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
 
                         this.setLastSegmentEntryId(metadataList.size());
                         this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList);
+                        List firstScheduleTimestamps = metadataList.stream().map(
+                                        SnapshotSegmentMetadata::getMinScheduleTimestamp).toList();
+                        this.setFirstScheduleTimestamps(firstScheduleTimestamps);
 
                         return nextSnapshotEntryIndex + 1;
                     });
@@ -157,8 +166,10 @@ CompletableFuture>
             return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId,
                     lastSegmentEntryId).whenComplete((__, ex) -> {
                 if (ex != null) {
-                    log.warn("[{}] Failed to get remain bucket snapshot segment, bucketKey: {}.",
-                            dispatcherName, bucketKey(), ex);
+                    log.warn(
+                            "[{}] Failed to get remain bucket snapshot segment, bucketKey: {},"
+                                    + " nextSegmentEntryId: {}, lastSegmentEntryId: {}",
+                            dispatcherName, bucketKey(), nextSegmentEntryId, lastSegmentEntryId, ex);
                 }
             });
         }, BucketSnapshotPersistenceException.class, MaxRetryTimes);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index e743f39e6920d..1577bf8fa51d2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -75,11 +75,15 @@ Pair createImmutableBucketAndAsyncPersistent(
         SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder();
         SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder();
 
+        List firstScheduleTimestamps = new ArrayList<>();
         long currentTimestampUpperLimit = 0;
+        long currentFirstTimestamp = 0L;
         while (!delayedIndexQueue.isEmpty()) {
             DelayedIndex delayedIndex = delayedIndexQueue.peek();
             long timestamp = delayedIndex.getTimestamp();
             if (currentTimestampUpperLimit == 0) {
+                currentFirstTimestamp = timestamp;
+                firstScheduleTimestamps.add(currentFirstTimestamp);
                 currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1;
             }
 
@@ -104,6 +108,7 @@ Pair createImmutableBucketAndAsyncPersistent(
                     || (maxIndexesPerBucketSnapshotSegment != -1
                     && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
                 segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp);
+                segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp);
                 currentTimestampUpperLimit = 0;
 
                 Iterator> iterator = bitMap.entrySet().iterator();
@@ -134,6 +139,7 @@ Pair createImmutableBucketAndAsyncPersistent(
         bucket.setCurrentSegmentEntryId(1);
         bucket.setNumberBucketDelayedMessages(numMessages);
         bucket.setLastSegmentEntryId(lastSegmentEntryId);
+        bucket.setFirstScheduleTimestamps(firstScheduleTimestamps);
 
         // Skip first segment, because it has already been loaded
         List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size());
diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
index 8414a583fe5b0..6996b860c5249 100644
--- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
+++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
@@ -31,6 +31,7 @@ message DelayedIndex {
 message SnapshotSegmentMetadata {
     map delayed_index_bit_map = 1;
     required uint64 max_schedule_timestamp = 2;
+    required uint64 min_schedule_timestamp = 3;
 }
 
 message SnapshotSegment {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
index 72052d22b859d..a628b58e10d32 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
@@ -73,6 +73,7 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException
     public void testGetSnapshot() throws ExecutionException, InterruptedException {
         DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
                 DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+                        .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
@@ -122,6 +123,7 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
         DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
                 DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
                         .setMaxScheduleTimestamp(timeMillis)
+                        .setMinScheduleTimestamp(timeMillis)
                         .putAllDelayedIndexBitMap(map).build();
 
         DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
@@ -172,6 +174,7 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException
     public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException {
         DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
                 DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+                        .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 74101f00b960c..0d53c278fd2f6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -52,6 +52,7 @@
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
 import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -251,18 +252,28 @@ public void testRoaringBitmapSerialize() {
     }
 
     @Test(dataProvider = "delayedTracker")
-    public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) {
+    public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) {
         for (int i = 1; i <= 110; i++) {
             tracker.addMessage(i, i, i * 10);
+            Awaitility.await().untilAsserted(() -> {
+                Assert.assertTrue(
+                        tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+            });
         }
 
         assertEquals(110, tracker.getNumberOfDelayedMessages());
 
         int size = tracker.getImmutableBuckets().asMapOfRanges().size();
 
-        assertEquals(10, size);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(10, size);
+        });
 
         tracker.addMessage(111, 1011, 111 * 10);
+        Awaitility.await().untilAsserted(() -> {
+            Assert.assertTrue(
+                    tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+        });
 
         MutableLong delayedMessagesInSnapshot = new MutableLong();
         tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> {
@@ -271,26 +282,28 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) {
 
         tracker.close();
 
-        tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
+        BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10);
 
-        assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue());
+        assertEquals(tracker2.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue());
 
         for (int i = 1; i <= 110; i++) {
-            tracker.addMessage(i, i, i * 10);
+            tracker2.addMessage(i, i, i * 10);
         }
 
         clockTime.set(110 * 10);
 
-        NavigableSet scheduledMessages = tracker.getScheduledMessages(110);
+        NavigableSet scheduledMessages = tracker2.getScheduledMessages(110);
         for (int i = 1; i <= 110; i++) {
             PositionImpl position = scheduledMessages.pollFirst();
             assertEquals(position, PositionImpl.get(i, i));
         }
+
+        tracker2.close();
     }
 
     @Test(dataProvider = "delayedTracker")
-    public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
+    public void testWithBkException(final BucketDelayedDeliveryTracker tracker) {
         MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage;
         mockBucketSnapshotStorage.injectCreateException(
                 new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry"));
@@ -308,11 +321,25 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
 
         for (int i = 1; i <= 110; i++) {
             tracker.addMessage(i, i, i * 10);
+            Awaitility.await().untilAsserted(() -> {
+                Assert.assertTrue(
+                        tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+            });
         }
 
         assertEquals(110, tracker.getNumberOfDelayedMessages());
 
+        int size = tracker.getImmutableBuckets().asMapOfRanges().size();
+
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(10, size);
+        });
+
         tracker.addMessage(111, 1011, 111 * 10);
+        Awaitility.await().untilAsserted(() -> {
+            Assert.assertTrue(
+                    tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+        });
 
         MutableLong delayedMessagesInSnapshot = new MutableLong();
         tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> {
@@ -321,11 +348,11 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
 
         tracker.close();
 
-        tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
+        BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
                 true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
 
         Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue();
-        assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue);
+        assertEquals(tracker2.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue);
 
         clockTime.set(110 * 10);
 
@@ -338,14 +365,16 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
         mockBucketSnapshotStorage.injectGetSegmentException(
                 new BucketSnapshotPersistenceException("Bookie operation timeout4, op: Get entry"));
 
-        assertEquals(tracker.getScheduledMessages(100).size(), 0);
+        assertEquals(tracker2.getScheduledMessages(100).size(), 0);
 
-        assertEquals(tracker.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue);
+        assertEquals(tracker2.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue);
 
         assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.getSegmentExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.deleteExceptionQueue.isEmpty());
+
+        tracker2.close();
     }
 
     @Test(dataProvider = "delayedTracker")
@@ -390,6 +419,8 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
         tracker.getImmutableBuckets().asMapOfRanges().forEach((k, bucket) -> {
             assertEquals(bucket.getLastSegmentEntryId(), 4);
         });
+
+        tracker.close();
     }
     
     @Test(dataProvider = "delayedTracker")
@@ -408,5 +439,7 @@ public void testClear(BucketDelayedDeliveryTracker tracker) {
       assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0);
       assertEquals(tracker.getLastMutableBucket().size(), 0);
       assertEquals(tracker.getSharedBucketPriorityQueue().size(), 0);
+
+      tracker.close();
     }
 }

From 329e80b847bd6584cbb031a96d5416d4df488432 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sun, 26 Mar 2023 00:30:40 +0800
Subject: [PATCH 107/404] [improve][broker] Clear delayed message when
 unsubscribe &  Make clear operation asynchronous (#19901)

---
 .../delayed/DelayedDeliveryTracker.java       |  5 +-
 .../InMemoryDelayedDeliveryTracker.java       |  4 +-
 .../pulsar/broker/delayed/bucket/Bucket.java  | 21 +++++--
 .../bucket/BucketDelayedDeliveryTracker.java  | 63 +++++++++++++------
 .../delayed/bucket/ImmutableBucket.java       | 57 +++--------------
 .../broker/delayed/bucket/MutableBucket.java  |  7 ++-
 .../pulsar/broker/service/Dispatcher.java     |  4 +-
 ...PersistentDispatcherMultipleConsumers.java | 23 ++++++-
 .../persistent/PersistentSubscription.java    | 11 +++-
 .../service/persistent/PersistentTopic.java   | 46 +++++++++++++-
 .../BucketDelayedDeliveryTrackerTest.java     |  7 ++-
 .../persistent/BucketDelayedDeliveryTest.java | 59 +++++++++++++++++
 12 files changed, 217 insertions(+), 90 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
index 2f248a441cdee..3cc2da8db1e4d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
@@ -20,6 +20,7 @@
 
 import com.google.common.annotations.Beta;
 import java.util.NavigableSet;
+import java.util.concurrent.CompletableFuture;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 
 /**
@@ -81,8 +82,10 @@ public interface DelayedDeliveryTracker extends AutoCloseable {
 
     /**
      * Clear all delayed messages from the tracker.
+     *
+     * @return CompletableFuture
      */
-    void clear();
+    CompletableFuture clear();
 
     /**
      * Close the subscription tracker and release all resources.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
index f55d5fd11694b..8de6ee58e2ce5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
@@ -23,6 +23,7 @@
 import java.time.Clock;
 import java.util.NavigableSet;
 import java.util.TreeSet;
+import java.util.concurrent.CompletableFuture;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
@@ -147,8 +148,9 @@ public NavigableSet getScheduledMessages(int maxMessages) {
     }
 
     @Override
-    public void clear() {
+    public CompletableFuture clear() {
         this.priorityQueue.clear();
+        return CompletableFuture.completedFuture(null);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index 7cfccff7ba328..5b7023be5034e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -33,6 +33,7 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
 import org.apache.pulsar.common.util.Codec;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.roaringbitmap.RoaringBitmap;
 
 @Slf4j
@@ -47,6 +48,9 @@ abstract class Bucket {
     protected final String dispatcherName;
 
     protected final ManagedCursor cursor;
+
+    protected final FutureUtil.Sequencer sequencer;
+
     protected final BucketSnapshotStorage bucketSnapshotStorage;
 
     long startLedgerId;
@@ -67,9 +71,10 @@ abstract class Bucket {
     private volatile CompletableFuture snapshotCreateFuture;
 
 
-    Bucket(String dispatcherName, ManagedCursor cursor,
+    Bucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer,
            BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) {
-        this(dispatcherName, cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null);
+        this(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0,
+                null, null);
     }
 
     boolean containsMessage(long ledgerId, long entryId) {
@@ -154,12 +159,16 @@ CompletableFuture asyncSaveBucketSnapshot(
 
     private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) {
         Objects.requireNonNull(bucketId);
-        return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)),
-                ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        return sequencer.sequential(() -> {
+            return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)),
+                    ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        });
     }
 
     protected CompletableFuture removeBucketCursorProperty(String bucketKey) {
-        return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey),
-                ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        return sequencer.sequential(() -> {
+            return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey),
+                    ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        });
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 1cf8bd2b20d17..39973928abf5d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -31,6 +31,7 @@
 import io.netty.util.Timeout;
 import io.netty.util.Timer;
 import java.time.Clock;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -117,19 +118,23 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
         this.sharedBucketPriorityQueue = new TripleLongPriorityQueue();
         this.immutableBuckets = TreeRangeMap.create();
         this.snapshotSegmentLastIndexTable = HashBasedTable.create();
-        this.lastMutableBucket = new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), bucketSnapshotStorage);
+        this.lastMutableBucket =
+                new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(),
+                        bucketSnapshotStorage);
         this.numberDelayedMessages = recoverBucketSnapshot();
     }
 
     private synchronized long recoverBucketSnapshot() throws RuntimeException {
         ManagedCursor cursor = this.lastMutableBucket.getCursor();
+        FutureUtil.Sequencer sequencer = this.lastMutableBucket.getSequencer();
         Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>();
         cursor.getCursorProperties().keySet().forEach(key -> {
             if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) {
                 String[] keys = key.split(DELIMITER);
                 checkArgument(keys.length == 3);
                 ImmutableBucket immutableBucket =
-                        new ImmutableBucket(dispatcher.getName(), cursor, this.lastMutableBucket.bucketSnapshotStorage,
+                        new ImmutableBucket(dispatcher.getName(), cursor, sequencer,
+                                this.lastMutableBucket.bucketSnapshotStorage,
                                 Long.parseLong(keys[1]), Long.parseLong(keys[2]));
                 putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId),
                         immutableBucket, toBeDeletedBucketMap);
@@ -274,7 +279,7 @@ private void afterCreateImmutableBucket(Pair immu
                         });
 
                         immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId);
-                        immutableBuckets.remove(
+                        immutableBuckets.asMapOfRanges().remove(
                                 Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId));
                         snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(),
                                 lastDelayedIndex.getTimestamp());
@@ -442,8 +447,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB
                                         return CompletableFuture.allOf(removeAFuture, removeBFuture);
                                     });
 
-                            immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
-                            immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
+                            Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges();
+                            immutableBucketMap.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
+                            immutableBucketMap.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
                         }
                     });
         });
@@ -525,14 +531,15 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                     }
 
                     if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) {
-                        immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
+                        immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
                         bucket.asyncDeleteBucketSnapshot();
                         continue;
                     }
 
                     bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> {
                         if (CollectionUtils.isEmpty(indexList)) {
-                            immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
+                            immutableBuckets.asMapOfRanges()
+                                    .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
                             bucket.asyncDeleteBucketSnapshot();
                             return;
                         }
@@ -549,13 +556,13 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                             // Back bucket state
                             bucket.setCurrentSegmentEntryId(preSegmentEntryId);
 
-                            log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}",
-                                    dispatcher.getName(), bucket.bucketKey(), ex);
+                            log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}",
+                                    dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex);
                         } else {
                             log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}",
-                                    dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId);
+                                    dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1);
                         }
-                    }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS);
+                    }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS);
                 } catch (Exception e) {
                     // Ignore exception to reload this segment on the next schedule.
                     log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}",
@@ -585,30 +592,46 @@ public boolean shouldPauseAllDeliveries() {
     }
 
     @Override
-    public synchronized void clear() {
-        cleanImmutableBuckets(true);
-        sharedBucketPriorityQueue.clear();
-        lastMutableBucket.clear();
-        snapshotSegmentLastIndexTable.clear();
-        numberDelayedMessages = 0;
+    public synchronized CompletableFuture clear() {
+        return cleanImmutableBuckets(true).thenRun(() -> {
+            synchronized (this) {
+                sharedBucketPriorityQueue.clear();
+                lastMutableBucket.clear();
+                snapshotSegmentLastIndexTable.clear();
+                numberDelayedMessages = 0;
+            }
+        });
     }
 
     @Override
     public synchronized void close() {
         super.close();
         lastMutableBucket.close();
-        cleanImmutableBuckets(false);
         sharedBucketPriorityQueue.close();
+        try {
+            cleanImmutableBuckets(false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e);
+        }
     }
 
-    private void cleanImmutableBuckets(boolean delete) {
+    private CompletableFuture cleanImmutableBuckets(boolean delete) {
         if (immutableBuckets != null) {
+            List> futures = new ArrayList<>();
             Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
             while (iterator.hasNext()) {
                 ImmutableBucket bucket = iterator.next();
-                bucket.clear(delete);
+                if (delete) {
+                    futures.add(bucket.clear());
+                } else {
+                    bucket.getSnapshotCreateFuture().ifPresent(future -> futures.add(future.thenApply(x -> null)));
+                }
+                numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
                 iterator.remove();
             }
+            return FutureUtil.waitForAll(futures);
+        } else {
+            return CompletableFuture.completedFuture(null);
         }
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index ab1c285011d20..969d326e28187 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.broker.delayed.bucket;
 
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
-import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds;
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE;
 import com.google.protobuf.ByteString;
 import java.util.ArrayList;
@@ -28,7 +27,6 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -38,6 +36,7 @@
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.roaringbitmap.RoaringBitmap;
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
 
@@ -52,9 +51,9 @@ class ImmutableBucket extends Bucket {
     @Setter
     List firstScheduleTimestamps = new ArrayList<>();
 
-    ImmutableBucket(String dispatcherName, ManagedCursor cursor,
+    ImmutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer,
                     BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) {
-        super(dispatcherName, cursor, storage, startLedgerId, endLedgerId);
+        super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId);
     }
 
     public Optional> getSnapshotSegments() {
@@ -117,8 +116,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
                     () -> bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId,
                             nextSegmentEntryId).whenComplete((___, ex) -> {
                         if (ex != null) {
-                            log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}, bucketId: {}",
-                                    dispatcherName, bucketKey(), bucketId, ex);
+                            log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {},"
+                                            + " bucketId: {}, segmentEntryId: {}", dispatcherName, bucketKey(),
+                                    bucketId, nextSegmentEntryId, ex);
                         }
                     }), BucketSnapshotPersistenceException.class, MaxRetryTimes)
                     .thenApply(bucketSnapshotSegments -> {
@@ -191,48 +191,9 @@ CompletableFuture asyncDeleteBucketSnapshot() {
         });
     }
 
-    void clear(boolean delete) {
+    CompletableFuture clear() {
         delayedIndexBitMap.clear();
-        if (delete) {
-            final String bucketKey = bucketKey();
-            try {
-                getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null).thenCompose(__ -> {
-                        if (getSnapshotCreateFuture().isPresent() && getBucketId().isEmpty()) {
-                            log.error("[{}] Can't found bucketId, don't execute delete operate, bucketKey: {}",
-                                    dispatcherName, bucketKey);
-                            return CompletableFuture.completedFuture(null);
-                        }
-                        long bucketId = getAndUpdateBucketId();
-                        return removeBucketCursorProperty(bucketKey()).thenAccept(___ -> {
-                            executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId),
-                            BucketSnapshotPersistenceException.class, MaxRetryTimes)
-                            .whenComplete((____, ex) -> {
-                                if (ex != null) {
-                                    log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
-                                            dispatcherName, bucketId, bucketKey, ex);
-                                } else {
-                                    log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
-                                            dispatcherName, bucketId, bucketKey);
-                                }
-                            });
-                    });
-                }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
-            } catch (Exception e) {
-                log.error("Failed to clear bucket snapshot, bucketKey: {}", bucketKey, e);
-                if (e instanceof InterruptedException) {
-                    Thread.currentThread().interrupt();
-                }
-                throw new RuntimeException(e);
-            }
-        } else {
-            getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> {
-                try {
-                    snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName,
-                            getBucketId(), bucketKey());
-                }
-            });
-        }
+        return getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null)
+                .thenCompose(__ -> asyncDeleteBucketSnapshot());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index 1577bf8fa51d2..f8a4ecc7a4ddb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 import org.roaringbitmap.RoaringBitmap;
 
@@ -42,9 +43,9 @@ class MutableBucket extends Bucket implements AutoCloseable {
 
     private final TripleLongPriorityQueue priorityQueue;
 
-    MutableBucket(String dispatcherName, ManagedCursor cursor,
+    MutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer,
                   BucketSnapshotStorage bucketSnapshotStorage) {
-        super(dispatcherName, cursor, bucketSnapshotStorage, -1L, -1L);
+        super(dispatcherName, cursor, sequencer, bucketSnapshotStorage, -1L, -1L);
         this.priorityQueue = new TripleLongPriorityQueue();
     }
 
@@ -134,7 +135,7 @@ Pair createImmutableBucketAndAsyncPersistent(
 
         final int lastSegmentEntryId = segmentMetadataList.size();
 
-        ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, bucketSnapshotStorage,
+        ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, sequencer, bucketSnapshotStorage,
                 startLedgerId, endLedgerId);
         bucket.setCurrentSegmentEntryId(1);
         bucket.setNumberBucketDelayedMessages(numMessages);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
index 48877ce53f801..9b0c4e885e64d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
@@ -110,8 +110,8 @@ default long getNumberOfDelayedMessages() {
         return 0;
     }
 
-    default void clearDelayedMessages() {
-        //No-op
+    default CompletableFuture clearDelayedMessages() {
+        return CompletableFuture.completedFuture(null);
     }
 
     default void cursorIsReset() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 059820b1b66c3..e5c9e85bac3f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -48,6 +48,7 @@
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
+import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
@@ -1089,8 +1090,26 @@ public synchronized long getNumberOfDelayedMessages() {
     }
 
     @Override
-    public void clearDelayedMessages() {
-        this.delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::clear);
+    public CompletableFuture clearDelayedMessages() {
+        if (!topic.isDelayedDeliveryEnabled()) {
+            return CompletableFuture.completedFuture(null);
+        }
+
+        if (delayedDeliveryTracker.isEmpty() && topic.getBrokerService()
+                .getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory) {
+            synchronized (this) {
+                if (delayedDeliveryTracker.isEmpty()) {
+                    delayedDeliveryTracker = Optional
+                            .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this));
+                }
+            }
+        }
+
+        if (delayedDeliveryTracker.isPresent()) {
+            return this.delayedDeliveryTracker.get().clear();
+        } else {
+            return CompletableFuture.completedFuture(null);
+        }
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index e07d7bee500a7..bdb3c9fc391ce 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -649,9 +649,16 @@ public void clearBacklogComplete(Object ctx) {
                             cursor.getNumberOfEntriesInBacklog(false));
                 }
                 if (dispatcher != null) {
-                    dispatcher.clearDelayedMessages();
+                    dispatcher.clearDelayedMessages().whenComplete((__, ex) -> {
+                        if (ex != null) {
+                            future.completeExceptionally(ex);
+                        } else {
+                            future.complete(null);
+                        }
+                    });
+                } else {
+                    future.complete(null);
                 }
-                future.complete(null);
             }
 
             @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index b3b6526eea54d..82a4f5312357a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -79,6 +79,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy;
 import org.apache.pulsar.broker.namespace.NamespaceService;
@@ -1078,13 +1079,13 @@ public CompletableFuture unsubscribe(String subscriptionName) {
                     new AsyncCallbacks.DeleteLedgerCallback() {
                         @Override
                         public void deleteLedgerComplete(Object ctx) {
-                            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+                            asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture);
                         }
 
                         @Override
                         public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) {
                             if (exception instanceof MetadataNotFoundException) {
-                                asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+                                asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture);
                                 return;
                             }
 
@@ -1094,12 +1095,51 @@ public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) {
                         }
                     }, null);
         } else {
-            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture);
         }
 
         return unsubscribeFuture;
     }
 
+    private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName,
+                                                          CompletableFuture unsubscribeFuture) {
+        if (!isDelayedDeliveryEnabled()
+                || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) {
+            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            return;
+        }
+
+        PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName);
+        if (persistentSubscription == null) {
+            log.warn("[{}][{}] Can't find subscription, skip clear delayed message", topic, subscriptionName);
+            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            return;
+        }
+
+        Dispatcher dispatcher = persistentSubscription.getDispatcher();
+        final Dispatcher temporaryDispatcher;
+        if (dispatcher == null) {
+            log.info("[{}][{}] Dispatcher is null, try to create temporary dispatcher to clear delayed message", topic,
+                    subscriptionName);
+            dispatcher = temporaryDispatcher =
+                    new PersistentDispatcherMultipleConsumers(this, persistentSubscription.cursor,
+                            persistentSubscription);
+        } else {
+            temporaryDispatcher = null;
+        }
+
+        dispatcher.clearDelayedMessages().whenComplete((__, ex) -> {
+            if (ex != null) {
+                unsubscribeFuture.completeExceptionally(ex);
+            } else {
+                asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            }
+            if (temporaryDispatcher != null) {
+                temporaryDispatcher.close();
+            }
+        });
+    }
+
     private void asyncDeleteCursor(String subscriptionName, CompletableFuture unsubscribeFuture) {
         ledger.asyncDeleteCursor(Codec.encode(subscriptionName), new DeleteCursorCallback() {
             @Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 0d53c278fd2f6..fbb866d48ed69 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -39,7 +39,9 @@
 import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
@@ -424,7 +426,8 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
     }
     
     @Test(dataProvider = "delayedTracker")
-    public void testClear(BucketDelayedDeliveryTracker tracker) {
+    public void testClear(BucketDelayedDeliveryTracker tracker)
+            throws ExecutionException, InterruptedException, TimeoutException {
       for (int i = 1; i <= 1001; i++) {
           tracker.addMessage(i, i, i * 10);
       }
@@ -433,7 +436,7 @@ public void testClear(BucketDelayedDeliveryTracker tracker) {
       assertTrue(tracker.getImmutableBuckets().asMapOfRanges().size() > 0);
       assertEquals(tracker.getLastMutableBucket().size(), 1);
 
-      tracker.clear();
+      tracker.clear().get(1, TimeUnit.MINUTES);
 
       assertEquals(tracker.getNumberOfDelayedMessages(), 0);
       assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 292889e8c159a..09b7cbbf1b99d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -18,9 +18,15 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
+import static org.testng.Assert.assertTrue;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
@@ -103,4 +109,57 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce
         Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000));
         Assert.assertEquals(bucketKeys, bucketKeys2);
     }
+
+
+    @Test
+    public void testUnsubscribe() throws Exception {
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testUnsubscribes");
+
+        @Cleanup
+        Consumer c1 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        for (int i = 0; i < 1000; i++) {
+            producer.newMessage()
+                    .value("msg")
+                    .deliverAfter(1, TimeUnit.HOURS)
+                    .send();
+        }
+
+        Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000));
+
+        Map cursorProperties =
+                ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties();
+        List bucketIds = cursorProperties.entrySet().stream()
+                .filter(x -> x.getKey().startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).map(
+                        x -> Long.valueOf(x.getValue())).toList();
+
+        assertTrue(bucketIds.size() > 0);
+
+        c1.close();
+
+        restartBroker();
+
+        admin.topics().deleteSubscription(topic, "sub");
+
+        for (Long bucketId : bucketIds) {
+            try {
+                LedgerHandle ledgerHandle =
+                        pulsarTestContext.getBookKeeperClient()
+                                .openLedger(bucketId, BookKeeper.DigestType.CRC32C, new byte[]{});
+                Assert.fail("Should fail");
+            } catch (BKException.BKNoSuchLedgerExistsException e) {
+                // ignore it
+            }
+        }
+    }
 }

From 2b4a3c14458f564e7e0178f71979a41e7b7a42b7 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Sun, 26 Mar 2023 22:36:34 +0800
Subject: [PATCH 108/404] [fix][broker] Fix RetentionPolicies constructor
 (#19777)

Signed-off-by: Zixuan Liu 
---
 .../apache/bookkeeper/mledger/ManagedLedgerConfig.java    | 8 ++++----
 .../pulsar/broker/service/ConsumedLedgersTrimTest.java    | 2 +-
 .../pulsar/common/policies/data/RetentionPolicies.java    | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index 6e88a8e650d58..0c93a5b642cf6 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -62,7 +62,7 @@ public class ManagedLedgerConfig {
     private int ledgerRolloverTimeout = 4 * 3600;
     private double throttleMarkDelete = 0;
     private long retentionTimeMs = 0;
-    private int retentionSizeInMB = 0;
+    private long retentionSizeInMB = 0;
     private boolean autoSkipNonRecoverableData;
     private boolean lazyCursorRecovery = false;
     private long metadataOperationsTimeoutSeconds = 60;
@@ -396,7 +396,7 @@ public ManagedLedgerConfig setThrottleMarkDelete(double throttleMarkDelete) {
     /**
      * Set the retention time for the ManagedLedger.
      * 

-     * Retention time and retention size ({@link #setRetentionSizeInMB(int)}) are together used to retain the
+     * Retention time and retention size ({@link #setRetentionSizeInMB(long)}) are together used to retain the
      * ledger data when there are no cursors or when all the cursors have marked the data for deletion.
      * Data will be deleted in this case when both retention time and retention size settings don't prevent deleting
      * the data marked for deletion.
@@ -438,7 +438,7 @@ public long getRetentionTimeMillis() {
      * @param retentionSizeInMB
      *            quota for message retention
      */
-    public ManagedLedgerConfig setRetentionSizeInMB(int retentionSizeInMB) {
+    public ManagedLedgerConfig setRetentionSizeInMB(long retentionSizeInMB) {
         this.retentionSizeInMB = retentionSizeInMB;
         return this;
     }
@@ -447,7 +447,7 @@ public ManagedLedgerConfig setRetentionSizeInMB(int retentionSizeInMB) {
      * @return quota for message retention
      *
      */
-    public int getRetentionSizeInMB() {
+    public long getRetentionSizeInMB() {
         return retentionSizeInMB;
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java
index 099a9028c4645..80db4c30f454d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java
@@ -86,7 +86,7 @@ public void TestConsumedLedgersTrim() throws Exception {
         PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
 
         ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig();
-        managedLedgerConfig.setRetentionSizeInMB(1);
+        managedLedgerConfig.setRetentionSizeInMB(1L);
         managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS);
         managedLedgerConfig.setMaxEntriesPerLedger(2);
         managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java
index 4206220c5ee58..8d5b25da43153 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java
@@ -29,13 +29,13 @@
  */
 public class RetentionPolicies {
     private int retentionTimeInMinutes;
-    private int retentionSizeInMB;
+    private long retentionSizeInMB;
 
     public RetentionPolicies() {
         this(0, 0);
     }
 
-    public RetentionPolicies(int retentionTimeInMinutes, int retentionSizeInMB) {
+    public RetentionPolicies(int retentionTimeInMinutes, long retentionSizeInMB) {
         this.retentionSizeInMB = retentionSizeInMB;
         this.retentionTimeInMinutes = retentionTimeInMinutes;
     }
@@ -44,7 +44,7 @@ public int getRetentionTimeInMinutes() {
         return retentionTimeInMinutes;
     }
 
-    public int getRetentionSizeInMB() {
+    public long getRetentionSizeInMB() {
         return retentionSizeInMB;
     }
 

From ef18bab1badbe6ce537254b3ff8fd288da1e7d4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tu=E1=BA=A5n=20V=C6=B0=C6=A1ng?= 
Date: Mon, 27 Mar 2023 11:59:50 +0700
Subject: [PATCH 109/404] [improve] Allow download link with basic auth
 (#19750)

Co-authored-by: tison 
---
 .../impl/auth/AuthenticationDataBasic.java    | 20 ++++++-----
 .../functions/utils/FunctionCommon.java       | 14 ++++++--
 .../functions/utils/FunctionCommonTest.java   | 35 ++++++++++++-------
 3 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java
index 2fc89e128ec58..82de6dc198d77 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java
@@ -27,16 +27,20 @@
 
 public class AuthenticationDataBasic implements AuthenticationDataProvider {
     private static final String HTTP_HEADER_NAME = "Authorization";
-    private String httpAuthToken;
-    private String commandAuthToken;
-    private Map headers = new HashMap<>();
+    private final String commandAuthToken;
+    private final Map headers;
 
     public AuthenticationDataBasic(String userId, String password) {
-        httpAuthToken = "Basic " + Base64.getEncoder().encodeToString((userId + ":" + password).getBytes());
-        commandAuthToken = userId + ":" + password;
-        headers.put(HTTP_HEADER_NAME, httpAuthToken);
-        headers.put(PULSAR_AUTH_METHOD_NAME, AuthenticationBasic.AUTH_METHOD_NAME);
-        this.headers = Collections.unmodifiableMap(this.headers);
+        this(userId + ":" + password);
+    }
+
+    public AuthenticationDataBasic(String userInfo) {
+        String httpAuthToken = "Basic " + Base64.getEncoder().encodeToString(userInfo.getBytes());
+        this.commandAuthToken = userInfo;
+        this.headers = Collections.unmodifiableMap(new HashMap(){{
+            put(HTTP_HEADER_NAME, httpAuthToken);
+            put(PULSAR_AUTH_METHOD_NAME, AuthenticationBasic.AUTH_METHOD_NAME);
+        }});
     }
 
     @Override
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index d3ce9d93a2d36..28cce0fe62209 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -35,9 +35,11 @@
 import java.net.ServerSocket;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.util.Collection;
+import java.util.Map;
 import java.util.UUID;
 import lombok.AccessLevel;
 import lombok.NoArgsConstructor;
@@ -48,6 +50,7 @@
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.Utils;
 import org.apache.pulsar.common.nar.NarClassLoader;
@@ -245,8 +248,15 @@ public static Class getSinkType(Class sinkClass) {
     }
 
     public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throws IOException {
-        URL website = new URL(destPkgUrl);
-        try (InputStream in = website.openStream()) {
+        final URL url = new URL(destPkgUrl);
+        final URLConnection connection = url.openConnection();
+        if (StringUtils.isNotEmpty(url.getUserInfo())) {
+            final AuthenticationDataBasic authBasic = new AuthenticationDataBasic(url.getUserInfo());
+            for (Map.Entry header : authBasic.getHttpHeaders()) {
+                connection.setRequestProperty(header.getKey(), header.getValue());
+            }
+        }
+        try (InputStream in = connection.getInputStream()) {
             log.info("Downloading function package from {} to {} ...", destPkgUrl, targetFile.getAbsoluteFile());
             Files.copy(in, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
         }
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
index aae2292752005..113824fc7c1a1 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
@@ -18,6 +18,11 @@
  */
 package org.apache.pulsar.functions.utils;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import java.io.File;
 import java.util.Collection;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -26,17 +31,11 @@
 import org.apache.pulsar.functions.api.Record;
 import org.apache.pulsar.functions.api.WindowContext;
 import org.apache.pulsar.functions.api.WindowFunction;
+import org.assertj.core.util.Files;
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
-import java.io.File;
-import java.util.UUID;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-
 /**
  * Unit test of {@link Exceptions}.
  */
@@ -78,12 +77,22 @@ public void testValidateHttpFileUrl() throws Exception {
 
     @Test
     public void testDownloadFile() throws Exception {
-        String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
-        String testDir = FunctionCommonTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-        File pkgFile = new File(testDir, UUID.randomUUID().toString());
-        FunctionCommon.downloadFromHttpUrl(jarHttpUrl, pkgFile);
-        Assert.assertTrue(pkgFile.exists());
-        pkgFile.delete();
+        final String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
+        final File file = Files.newTemporaryFile();
+        file.deleteOnExit();
+        assertThat(file.length()).isZero();
+        FunctionCommon.downloadFromHttpUrl(jarHttpUrl, file);
+        assertThat(file.length()).isGreaterThan(0);
+    }
+
+    @Test
+    public void testDownloadFileWithBasicAuth() throws Exception {
+        final String jarHttpUrl = "https://foo:bar@httpbin.org/basic-auth/foo/bar";
+        final File file = Files.newTemporaryFile();
+        file.deleteOnExit();
+        assertThat(file.length()).isZero();
+        FunctionCommon.downloadFromHttpUrl(jarHttpUrl, file);
+        assertThat(file.length()).isGreaterThan(0);
     }
 
     @Test

From aac99c86ea39ad673bce32240056fe308f546bef Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Mon, 27 Mar 2023 14:24:18 +0800
Subject: [PATCH 110/404] [fix][client] Fix NPE when acknowledging multiple
 messages (#19874)

---
 .../client/api/ConsumerAckListTest.java       | 39 +++++++++++++++++++
 .../client/impl/MultiTopicsConsumerImpl.java  | 13 ++++++-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java
index d01e5b764a7a1..baf0000be6e4f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java
@@ -111,4 +111,43 @@ private void sendMessagesAsyncAndWait(Producer producer, int messages) t
         latch.await();
     }
 
+    @Test(timeOut = 30000)
+    public void testAckMessageInAnotherTopic() throws Exception {
+        final String[] topics = {
+                "persistent://my-property/my-ns/test-ack-message-in-other-topic1" + UUID.randomUUID(),
+                "persistent://my-property/my-ns/test-ack-message-in-other-topic2" + UUID.randomUUID(),
+                "persistent://my-property/my-ns/test-ack-message-in-other-topic3" + UUID.randomUUID()
+        };
+        @Cleanup final Consumer allTopicsConsumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topics)
+                .subscriptionName("sub1")
+                .subscribe();
+        Consumer partialTopicsConsumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topics[0], topics[1])
+                .subscriptionName("sub2")
+                .subscribe();
+        for (int i = 0; i < topics.length; i++) {
+            final Producer producer = pulsarClient.newProducer(Schema.STRING)
+                    .topic(topics[i])
+                    .create();
+            producer.send("msg-" + i);
+            producer.close();
+        }
+        final List messageIdList = new ArrayList<>();
+        for (int i = 0; i < topics.length; i++) {
+            messageIdList.add(allTopicsConsumer.receive().getMessageId());
+        }
+        try {
+            partialTopicsConsumer.acknowledge(messageIdList);
+            Assert.fail();
+        } catch (PulsarClientException.NotConnectedException ignored) {
+        }
+        partialTopicsConsumer.close();
+        partialTopicsConsumer = pulsarClient.newConsumer(Schema.STRING).topic(topics[0])
+                .subscriptionName("sub2").subscribe();
+        pulsarClient.newProducer(Schema.STRING).topic(topics[0]).create().send("done");
+        final Message msg = partialTopicsConsumer.receive();
+        Assert.assertEquals(msg.getValue(), "msg-0");
+        partialTopicsConsumer.close();
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index 341a91e97348e..f993304b0780a 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -32,6 +32,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -494,8 +495,16 @@ protected CompletableFuture doAcknowledge(List messageIdList,
                 topicToMessageIdMap.get(topicMessageId.getOwnerTopic())
                         .add(MessageIdImpl.convertToMessageIdImpl(topicMessageId));
             }
-            topicToMessageIdMap.forEach((topicPartitionName, messageIds) -> {
-                ConsumerImpl consumer = consumers.get(topicPartitionName);
+            final Map, List> consumerToMessageIds = new IdentityHashMap<>();
+            for (Map.Entry> entry : topicToMessageIdMap.entrySet()) {
+                ConsumerImpl consumer = consumers.get(entry.getKey());
+                if (consumer == null) {
+                    return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException());
+                }
+                // Trigger the acknowledgment later to avoid sending partial acknowledgments
+                consumerToMessageIds.put(consumer, entry.getValue());
+            }
+            consumerToMessageIds.forEach((consumer, messageIds) -> {
                 resultFutures.add(consumer.doAcknowledgeWithTxn(messageIds, ackType, properties, txn)
                         .thenAccept((res) -> messageIdList.forEach(unAckedMessageTracker::remove)));
             });

From 19c84977f00033ef9601f43f1b703068a6207536 Mon Sep 17 00:00:00 2001
From: LinChen 
Date: Mon, 27 Mar 2023 15:18:01 +0800
Subject: [PATCH 111/404] [fix][broker] Fix the thread safety issue of
 BrokerData#getTimeAverageData access (#19889)

Co-authored-by: lordcheng10 
---
 .../broker/loadbalance/impl/ModularLoadManagerImpl.java       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index f135840d60e59..4756b885ff217 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -589,7 +589,9 @@ private void updateBundleData() {
             }
 
             // Using the newest data, update the aggregated time-average data for the current broker.
-            brokerData.getTimeAverageData().reset(statsMap.keySet(), bundleData, defaultStats);
+            TimeAverageBrokerData timeAverageData = new TimeAverageBrokerData();
+            timeAverageData.reset(statsMap.keySet(), bundleData, defaultStats);
+            brokerData.setTimeAverageData(timeAverageData);
             final ConcurrentOpenHashMap> namespaceToBundleRange =
                     brokerToNamespaceToBundleRange
                             .computeIfAbsent(broker, k ->

From cda2827acbe62ba6c5a4dd98dbee7c9a82548339 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 27 Mar 2023 12:32:54 +0100
Subject: [PATCH 112/404] [fix][fn] Revert change to deprecation since it broke
 the master branch (#19904)

---
 pulsar-function-go/pb/Function.pb.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pulsar-function-go/pb/Function.pb.go b/pulsar-function-go/pb/Function.pb.go
index bad1afe6658da..3e4d49ab99022 100644
--- a/pulsar-function-go/pb/Function.pb.go
+++ b/pulsar-function-go/pb/Function.pb.go
@@ -579,7 +579,6 @@ type FunctionDetails struct {
 	Runtime              FunctionDetails_Runtime `protobuf:"varint,8,opt,name=runtime,proto3,enum=proto.FunctionDetails_Runtime" json:"runtime,omitempty"`
 	// Deprecated since, see https://github.com/apache/pulsar/issues/15560
 	//
-	// Deprecated: Do not use.
 	AutoAck              bool                          `protobuf:"varint,9,opt,name=autoAck,proto3" json:"autoAck,omitempty"`
 	Parallelism          int32                         `protobuf:"varint,10,opt,name=parallelism,proto3" json:"parallelism,omitempty"`
 	Source               *SourceSpec                   `protobuf:"bytes,11,opt,name=source,proto3" json:"source,omitempty"`

From 32ad90606062c1eda660037e6277253b35d4a1e6 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Mon, 27 Mar 2023 22:16:25 +0800
Subject: [PATCH 113/404] [fix][admin] Delete tenant local policy only if exist
 (#19925)

Signed-off-by: nodece 
---
 .../resources/LocalPoliciesResources.java     |  2 +-
 .../broker/admin/AdminApiTenantTest.java      | 76 +++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java
index 8ca0c121ef1b9..c6b658c3bd025 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java
@@ -85,7 +85,7 @@ public CompletableFuture deleteLocalPoliciesAsync(NamespaceName ns) {
     public CompletableFuture deleteLocalPoliciesTenantAsync(String tenant) {
         final String localPoliciesPath = joinPath(LOCAL_POLICIES_ROOT, tenant);
         CompletableFuture future = new CompletableFuture();
-        deleteAsync(localPoliciesPath).whenComplete((ignore, ex) -> {
+        deleteIfExistsAsync(localPoliciesPath).whenComplete((ignore, ex) -> {
             if (ex != null && ex.getCause().getCause() instanceof KeeperException) {
                 future.complete(null);
             } else if (ex != null) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java
new file mode 100644
index 0000000000000..f883417614229
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-admin")
+@Slf4j
+public class AdminApiTenantTest extends MockedPulsarServiceBaseTest {
+    private final String CLUSTER = "test";
+
+    @BeforeClass
+    @Override
+    public void setup() throws Exception {
+        super.internalSetup();
+        admin.clusters()
+                .createCluster(CLUSTER, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
+    }
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    public void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @Test
+    public void testListTenant() throws PulsarAdminException {
+        admin.tenants().getTenants();
+    }
+
+    @Test
+    public void testCreateAndDeleteTenant() throws PulsarAdminException {
+        String tenant = "test-tenant-"+ UUID.randomUUID();
+        admin.tenants().createTenant(tenant, TenantInfo.builder().allowedClusters(Collections.singleton(CLUSTER)).build());
+        List tenants = admin.tenants().getTenants();
+        assertTrue(tenants.contains(tenant));
+        admin.tenants().deleteTenant(tenant);
+        tenants = admin.tenants().getTenants();
+        assertFalse(tenants.contains(tenant));
+    }
+
+    @Test
+    public void testDeleteNonExistTenant() {
+        String tenant = "test-non-exist-tenant-" + UUID.randomUUID();
+        assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.tenants().deleteTenant(tenant));
+    }
+}

From d14e43ed9bd079b76301f10dc1c039cafd43e1cf Mon Sep 17 00:00:00 2001
From: tison 
Date: Tue, 28 Mar 2023 11:22:22 +0800
Subject: [PATCH 114/404] [improve][client] Exclude log4j-slf4j-impl from
 compile dep in pulsar-client-all (#19937)

Signed-off-by: tison 
---
 pulsar-client-all/pom.xml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index b06c71616b6a5..00da6e4895097 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -54,11 +54,25 @@
       ${project.parent.version}
       true
     
+    
+      org.apache.logging.log4j
+      log4j-api
+      test
+    
+    
+      org.apache.logging.log4j
+      log4j-core
+      test
+    
+    
+      org.apache.logging.log4j
+      log4j-slf4j-impl
+      test
+    
   
 
   
     
-
       
         org.apache.maven.plugins
         maven-dependency-plugin

From 7a99e74220bdff7a0dff3dbbf3befb2316f3a6c8 Mon Sep 17 00:00:00 2001
From: SeasonPan <244014926@qq.com>
Date: Tue, 28 Mar 2023 17:58:58 +0800
Subject: [PATCH 115/404] [fix][doc] fix typo in ConsumerConfigurationData and
 ServiceConfiguration. (#19936)

---
 .../java/org/apache/pulsar/broker/ServiceConfiguration.java   | 2 +-
 .../pulsar/client/impl/conf/ConsumerConfigurationData.java    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ba82667690dc5..e2bcf03353b59 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1976,7 +1976,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
             category = CATEGORY_STORAGE_ML,
             dynamic = true,
             doc = "The number of partitioned topics that is allowed to be automatically created"
-                    + "if allowAutoTopicCreationType is partitioned."
+                    + " if allowAutoTopicCreationType is partitioned."
     )
     private int defaultNumPartitions = 1;
     @FieldContext(
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java
index 373d4e66c0ecf..8760926792cd7 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java
@@ -99,7 +99,7 @@ public class ConsumerConfigurationData implements Serializable, Cloneable {
     @ApiModelProperty(
             name = "negativeAckRedeliveryBackoff",
             value = "Interface for custom message is negativeAcked policy. You can specify `RedeliveryBackoff` for a"
-                    + "consumer."
+                    + " consumer."
     )
     @JsonIgnore
     private RedeliveryBackoff negativeAckRedeliveryBackoff;
@@ -235,7 +235,7 @@ public int getMaxPendingChuckedMessage() {
 
     @ApiModelProperty(
             name = "autoAckOldestChunkedMessageOnQueueFull",
-            value = "Whether to automatically acknowledge pending chunked messages when the threashold of"
+            value = "Whether to automatically acknowledge pending chunked messages when the threshold of"
                     + " `maxPendingChunkedMessage` is reached. If set to `false`, these messages will be redelivered"
                     + " by their broker."
     )

From 83f6ea74e7a47927402d8f1de4200c4ad02e6147 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 28 Mar 2023 19:07:43 +0100
Subject: [PATCH 116/404] [fix][build] Dump Jacoco coverage data to file with
 JMX interface in TestNG listener (#19947)

---
 buildtools/pom.xml                            |   2 +-
 .../pulsar/tests/JacocoDumpListener.java      | 103 ++++++++++++++++++
 pom.xml                                       |   3 +-
 3 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index de52ac0930a33..b22abf286f693 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -176,7 +176,7 @@
           
             
               listener
-              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier
+              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier
             
           
           
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java
new file mode 100644
index 0000000000000..2c49d5118ae52
--- /dev/null
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeUnit;
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import org.testng.IExecutionListener;
+import org.testng.ISuite;
+import org.testng.ISuiteListener;
+
+/**
+ * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface.
+ *
+ * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage
+ * data will be dumped every 2 minutes by default and once all test suites have been run.
+ * Each test class runs in its own suite when run with maven-surefire-plugin.
+ */
+public class JacocoDumpListener implements ISuiteListener, IExecutionListener {
+    private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
+    private final ObjectName jacocoObjectName;
+    private final JacocoProxy jacocoProxy;
+    private final boolean enabled;
+
+    private long lastDumpTime;
+
+    private static final long DUMP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(120);
+
+    public JacocoDumpListener() {
+        try {
+            jacocoObjectName = new ObjectName("org.jacoco:type=Runtime");
+        } catch (MalformedObjectNameException e) {
+            // this won't happen since the ObjectName is static and valid
+            throw new RuntimeException(e);
+        }
+        enabled = checkEnabled();
+        if (enabled) {
+            jacocoProxy = MBeanServerInvocationHandler.newProxyInstance(platformMBeanServer, jacocoObjectName,
+                    JacocoProxy.class, false);
+        } else {
+            jacocoProxy = null;
+        }
+        lastDumpTime = System.currentTimeMillis();
+    }
+
+    private boolean checkEnabled() {
+        try {
+            platformMBeanServer.getObjectInstance(jacocoObjectName);
+        } catch (InstanceNotFoundException e) {
+            // jacoco jmx is not enabled
+            return false;
+        }
+        return true;
+    }
+
+    public void onFinish(ISuite suite) {
+        // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed
+        // since the last dump
+        if (enabled && System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS) {
+            // dump jacoco coverage data to file using the Jacoco JMX interface
+            triggerJacocoDump();
+        }
+    }
+    @Override
+    public void onExecutionFinish() {
+        if (enabled) {
+            // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished
+            triggerJacocoDump();
+        }
+    }
+
+    private void triggerJacocoDump() {
+        System.out.println("Dumping Jacoco coverage data to file...");
+        long start = System.currentTimeMillis();
+        jacocoProxy.dump(true);
+        lastDumpTime = System.currentTimeMillis();
+        System.out.println("Completed in " + (lastDumpTime - start) + "ms.");
+    }
+
+    public interface JacocoProxy {
+        void dump(boolean reset);
+    }
+}
diff --git a/pom.xml b/pom.xml
index 3f33069b8c252..ac640f1a6a05f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1531,7 +1531,7 @@ flexible messaging model and an intuitive client API.
           
             
               listener
-              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener
+              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener
             
           
         
@@ -1995,6 +1995,7 @@ flexible messaging model and an intuitive client API.
                   
                   ${project.build.directory}/jacoco_${maven.build.timestamp}_${surefire.forkNumber}.exec
                   true
+                  true
                   
                     org.apache.pulsar.*
                     org.apache.bookkeeper.mledger.*

From 9fc0b5e197aa36ff64087cd13d156bb65a097fad Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 29 Mar 2023 09:24:12 +0800
Subject: [PATCH 117/404] [fix] [admin] fix incorrect state
 replication.connected on API partitioned-topic stat (#19942)

### Motivation

Pulsar will merge the variable `PartitionedTopicStatsImpl.replication[x].connected` by the way below when we call `pulsar-admin topics partitioned-stats`

``` java
this.connected = this.connected & other.connected
```

But the variable `connected` of `PartitionedTopicStatsImpl.replication` is initialized `false`, so the expression `this.connected & other.connected` will always be `false`.

Then we will always get the value `false` if we call `pulsar-admin topics partitioned-stats`.

### Modifications

make the variable `` of `PartitionedTopicStatsImpl` is initialized `true`
---
 .../pulsar/broker/service/ReplicatorTest.java | 25 +++++++++++++++++++
 .../data/stats/ReplicatorStatsImpl.java       |  4 ++-
 .../policies/data/stats/TopicStatsImpl.java   |  2 ++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index 4086c54a0ba4b..901451c022bfe 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -97,8 +97,10 @@
 import org.apache.pulsar.common.policies.data.BacklogQuota;
 import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy;
 import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
 import org.apache.pulsar.common.policies.data.ReplicatorStats;
 import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.schema.Schemas;
@@ -870,6 +872,29 @@ public void testReplicatorProducerClosing() throws Exception {
         assertNull(producer);
     }
 
+    @Test(priority = 5, timeOut = 30000)
+    public void testReplicatorConnected() throws Exception {
+        final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
+        final TopicName dest = TopicName.get(topicName);
+        admin1.topics().createPartitionedTopic(topicName, 1);
+
+        @Cleanup
+        MessageProducer producer1 = new MessageProducer(url1, dest);
+
+        Awaitility.await().until(() -> {
+            TopicStats topicStats = admin1.topics().getStats(topicName + "-partition-0");
+            return topicStats.getReplication().values().stream()
+                    .map(ReplicatorStats::isConnected).reduce((a, b) -> a & b).get();
+        });
+
+        PartitionedTopicStats
+                partitionedTopicStats = admin1.topics().getPartitionedStats(topicName, true);
+
+        for (ReplicatorStats replicatorStats : partitionedTopicStats.getReplication().values()){
+            assertTrue(replicatorStats.isConnected());
+        }
+    }
+
     @Test
     public void testDeleteTopicFailure() throws Exception {
         final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
index 71196e4978682..6933f5cc7ed76 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
@@ -72,7 +72,9 @@ public ReplicatorStatsImpl add(ReplicatorStatsImpl stats) {
         this.msgThroughputOut += stats.msgThroughputOut;
         this.msgRateExpired += stats.msgRateExpired;
         this.replicationBacklog += stats.replicationBacklog;
-        this.connected &= stats.connected;
+        if (this.connected) {
+            this.connected &= stats.connected;
+        }
         this.replicationDelayInSeconds = Math.max(this.replicationDelayInSeconds, stats.replicationDelayInSeconds);
         return this;
     }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index 238170952f08e..12d30124f7dcd 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -287,6 +287,7 @@ public TopicStatsImpl add(TopicStats ts) {
         if (this.replication.size() != stats.replication.size()) {
             for (String repl : stats.replication.keySet()) {
                 ReplicatorStatsImpl replStats = new ReplicatorStatsImpl();
+                replStats.setConnected(true);
                 this.replication.put(repl, replStats.add(stats.replication.get(repl)));
             }
         } else {
@@ -295,6 +296,7 @@ public TopicStatsImpl add(TopicStats ts) {
                     this.replication.get(repl).add(stats.replication.get(repl));
                 } else {
                     ReplicatorStatsImpl replStats = new ReplicatorStatsImpl();
+                    replStats.setConnected(true);
                     this.replication.put(repl, replStats.add(stats.replication.get(repl)));
                 }
             }

From eedf7026ae60f39bcf74ce67728b47d966fe237f Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Wed, 29 Mar 2023 10:23:10 +0800
Subject: [PATCH 118/404] [improve][broker] PIP-192: Support delete namespace
 bundle admin API (#19851)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

We need to support delete namespace bundle admin API.

### Modifications

* Support delete namespace bundle admin API.
* Add units test.
---
 .../broker/admin/impl/NamespacesBase.java     | 12 ++---
 .../extensions/ExtensibleLoadManagerImpl.java | 11 +++-
 .../extensions/data/BrokerLookupData.java     |  6 +++
 .../broker/namespace/NamespaceService.java    | 47 ++++++++++++++---
 .../ExtensibleLoadManagerImplTest.java        | 50 +++++++++++++++++++
 5 files changed, 111 insertions(+), 15 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index cffc94b189225..19f8d7c437ded 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -297,11 +297,11 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime
                 .thenCompose(ignore -> pulsar().getNamespaceService()
                         .getNamespaceBundleFactory().getBundlesAsync(namespaceName))
                 .thenCompose(bundles -> FutureUtil.waitForAll(bundles.getBundles().stream()
-                        .map(bundle -> pulsar().getNamespaceService().getOwnerAsync(bundle)
-                                .thenCompose(owner -> {
+                        .map(bundle -> pulsar().getNamespaceService().checkOwnershipPresentAsync(bundle)
+                                .thenCompose(present -> {
                                     // check if the bundle is owned by any broker,
                                     // if not then we do not need to delete the bundle
-                                    if (owner.isPresent()) {
+                                    if (present) {
                                         PulsarAdmin admin;
                                         try {
                                             admin = pulsar().getAdminClient();
@@ -1411,7 +1411,7 @@ protected void internalClearNamespaceBacklog(AsyncResponse asyncResponse, boolea
                     .getBundles(namespaceName);
             for (NamespaceBundle nsBundle : bundles.getBundles()) {
                 // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear
-                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
+                if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) {
                     futures.add(pulsar().getAdminClient().namespaces()
                             .clearNamespaceBundleBacklogAsync(namespaceName.toString(), nsBundle.getBundleRange()));
                 }
@@ -1476,7 +1476,7 @@ protected void internalClearNamespaceBacklogForSubscription(AsyncResponse asyncR
                     .getBundles(namespaceName);
             for (NamespaceBundle nsBundle : bundles.getBundles()) {
                 // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear
-                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
+                if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) {
                     futures.add(pulsar().getAdminClient().namespaces().clearNamespaceBundleBacklogForSubscriptionAsync(
                             namespaceName.toString(), nsBundle.getBundleRange(), subscription));
                 }
@@ -1543,7 +1543,7 @@ protected void internalUnsubscribeNamespace(AsyncResponse asyncResponse, String
                     .getBundles(namespaceName);
             for (NamespaceBundle nsBundle : bundles.getBundles()) {
                 // check if the bundle is owned by any broker, if not then there are no subscriptions
-                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
+                if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) {
                     futures.add(pulsar().getAdminClient().namespaces().unsubscribeNamespaceBundleAsync(
                             namespaceName.toString(), nsBundle.getBundleRange(), subscription));
                 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index fedcad4d009b2..7aefef596e7cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -383,7 +383,7 @@ public CompletableFuture checkOwnershipAsync(Optional to
                 .thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null)));
     }
 
-    private CompletableFuture> getOwnershipAsync(Optional topic,
+    public CompletableFuture> getOwnershipAsync(Optional topic,
                                                                  ServiceUnitId bundleUnit) {
         final String bundle = bundleUnit.toString();
         CompletableFuture> owner;
@@ -395,6 +395,15 @@ private CompletableFuture> getOwnershipAsync(Optional> getOwnershipWithLookupDataAsync(ServiceUnitId bundleUnit) {
+        return getOwnershipAsync(Optional.empty(), bundleUnit).thenCompose(broker -> {
+            if (broker.isEmpty()) {
+                return CompletableFuture.completedFuture(Optional.empty());
+            }
+            return getBrokerRegistry().lookupAsync(broker.get());
+        });
+    }
+
     public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                                                               Optional destinationBroker) {
         return getOwnershipAsync(Optional.empty(), bundle)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
index 504ae13003e04..4c9e503129e66 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 import java.util.Optional;
 import org.apache.pulsar.broker.lookup.LookupResult;
+import org.apache.pulsar.broker.namespace.NamespaceEphemeralData;
 import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
 import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
 
@@ -70,4 +71,9 @@ public LookupResult toLookupResult() {
         return new LookupResult(webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls,
                 LookupResult.Type.BrokerUrl, false);
     }
+
+    public NamespaceEphemeralData toNamespaceEphemeralData() {
+        return new NamespaceEphemeralData(pulsarServiceUrl, pulsarServiceUrlTls, webServiceUrl, webServiceUrlTls,
+                false, advertisedListeners);
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index de18d50f3e144..d092ef04018c8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1165,8 +1165,14 @@ public CompletableFuture checkTopicOwnership(TopicName topicName) {
     }
 
     public CompletableFuture removeOwnedServiceUnitAsync(NamespaceBundle nsBundle) {
-        return ownershipCache.removeOwnership(nsBundle)
-                .thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject()));
+        CompletableFuture future;
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get());
+            future = extensibleLoadManager.unloadNamespaceBundleAsync(nsBundle, Optional.empty());
+        } else {
+            future = ownershipCache.removeOwnership(nsBundle);
+        }
+        return future.thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject()));
     }
 
     protected void onNamespaceBundleOwned(NamespaceBundle bundle) {
@@ -1445,15 +1451,40 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) {
         });
     }
 
-    public Optional getOwner(NamespaceBundle bundle) throws Exception {
-        // if there is no znode for the service unit, it is not owned by any broker
-        return getOwnerAsync(bundle).get(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-    }
-
     public CompletableFuture> getOwnerAsync(NamespaceBundle bundle) {
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get());
+            return extensibleLoadManager.getOwnershipWithLookupDataAsync(bundle)
+                    .thenCompose(lookupData -> {
+                        if (lookupData.isPresent()) {
+                            return CompletableFuture.completedFuture(
+                                    Optional.of(lookupData.get().toNamespaceEphemeralData()));
+                        } else {
+                            return CompletableFuture.completedFuture(Optional.empty());
+                        }
+                    });
+        }
         return ownershipCache.getOwnerAsync(bundle);
     }
 
+    public boolean checkOwnershipPresent(NamespaceBundle bundle) throws Exception {
+        return checkOwnershipPresentAsync(bundle).get(pulsar.getConfiguration()
+                        .getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+    }
+
+    public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bundle) {
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            if (bundle.getNamespaceObject().equals(SYSTEM_NAMESPACE)) {
+                return FutureUtil.failedFuture(new UnsupportedOperationException(
+                        "Ownership check for system namespace is not supported"));
+            }
+            ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get());
+            return extensibleLoadManager.getOwnershipAsync(Optional.empty(), bundle)
+                    .thenApply(Optional::isPresent);
+        }
+        return getOwnerAsync(bundle).thenApply(Optional::isPresent);
+    }
+
     public void unloadSLANamespace() throws Exception {
         PulsarAdmin adminClient = null;
         NamespaceName namespaceName = getSLAMonitorNamespace(host, config);
@@ -1461,7 +1492,7 @@ public void unloadSLANamespace() throws Exception {
         LOG.info("Checking owner for SLA namespace {}", namespaceName);
 
         NamespaceBundle nsFullBundle = getFullBundle(namespaceName);
-        if (!getOwner(nsFullBundle).isPresent()) {
+        if (!checkOwnershipPresent(nsFullBundle)) {
             // No one owns the namespace so no point trying to unload it
             // Next lookup will assign the bundle to this broker.
             return;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index d977e63330586..f8a7a9b629f4e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -134,6 +134,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest {
     @BeforeClass
     @Override
     public void setup() throws Exception {
+        conf.setForceDeleteNamespaceAllowed(true);
         conf.setAllowAutoTopicCreation(true);
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
@@ -142,6 +143,7 @@ public void setup() throws Exception {
         pulsar1 = pulsar;
         ServiceConfiguration defaultConf = getDefaultConf();
         defaultConf.setAllowAutoTopicCreation(true);
+        defaultConf.setForceDeleteNamespaceAllowed(true);
         defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
         defaultConf.setLoadBalancerSheddingEnabled(false);
@@ -433,6 +435,54 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception {
         assertTrue(bundlesData.getBoundaries().contains(midBundle));
         assertTrue(bundlesData.getBoundaries().contains(highBundle));
     }
+    @Test(timeOut = 30 * 1000)
+    public void testDeleteNamespaceBundle() throws Exception {
+        TopicName topicName = TopicName.get("test-delete-namespace-bundle");
+        NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
+
+        String broker = admin.lookups().lookupTopic(topicName.toString());
+        log.info("Assign the bundle {} to {}", bundle, broker);
+
+        checkOwnershipState(broker, bundle);
+
+        admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange());
+        assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testDeleteNamespace() throws Exception {
+        String namespace = "public/test-delete-namespace";
+        TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-topic");
+        admin.namespaces().createNamespace(namespace);
+        admin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(this.conf.getClusterName()));
+        assertTrue(admin.namespaces().getNamespaces("public").contains(namespace));
+        admin.topics().createPartitionedTopic(topicName.toString(), 2);
+        admin.lookups().lookupTopic(topicName.toString());
+        NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
+        try {
+            admin.namespaces().deleteNamespaceBundle(namespace, bundle.getBundleRange());
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getMessage().contains("Cannot delete non empty bundle"));
+        }
+        admin.namespaces().deleteNamespaceBundle(namespace, bundle.getBundleRange(), true);
+        admin.lookups().lookupTopic(topicName.toString());
+
+        admin.namespaces().deleteNamespace(namespace, true);
+        assertFalse(admin.namespaces().getNamespaces("public").contains(namespace));
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testCheckOwnershipPresentWithSystemNamespace() throws Exception {
+        NamespaceBundle namespaceBundle =
+                getBundleAsync(pulsar1, TopicName.get(NamespaceName.SYSTEM_NAMESPACE + "/test")).get();
+        try {
+            pulsar1.getNamespaceService().checkOwnershipPresent(namespaceBundle);
+        } catch (Exception ex) {
+            log.info("Got exception", ex);
+            assertTrue(ex.getCause() instanceof UnsupportedOperationException);
+        }
+    }
 
     @Test
     public void testMoreThenOneFilter() throws Exception {

From d8774671f5d52be4ef940ea894ba4f9b1f5717f9 Mon Sep 17 00:00:00 2001
From: StevenLuMT 
Date: Wed, 29 Mar 2023 10:36:23 +0800
Subject: [PATCH 119/404] [fix][test] Shutdown executor on PerformanceProducer
 closed (#19926)

Co-authored-by: lushiji 
---
 .../pulsar/testclient/PerformanceProducer.java      | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index 9aedc8c4bd327..0c56f1d5c736d 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -356,6 +356,7 @@ public static void main(String[] args) throws Exception {
         long start = System.nanoTime();
 
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            executorShutdownNow();
             printAggregatedThroughput(start, arguments);
             printAggregatedStats();
         }));
@@ -481,6 +482,18 @@ public static void main(String[] args) throws Exception {
         }
     }
 
+    private static void executorShutdownNow() {
+        executor.shutdownNow();
+        try {
+            if (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
+                log.warn("Failed to terminate executor within timeout. The following are stack"
+                        + " traces of still running threads.");
+            }
+        } catch (InterruptedException e) {
+            log.warn("Shutdown of thread pool was interrupted");
+        }
+    }
+
     static IMessageFormatter getMessageFormatter(String formatterClass) {
         try {
             ClassLoader classLoader = PerformanceProducer.class.getClassLoader();

From 5611faf61e70ce563bb43eb499436517b54d0bb5 Mon Sep 17 00:00:00 2001
From: crossoverJie 
Date: Wed, 29 Mar 2023 13:06:02 +0800
Subject: [PATCH 120/404] [fix][client] Fix client erroneous code comments
 (#19915)

Co-authored-by: tison 
---
 .../pulsar/client/api/ClientBuilder.java      | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java
index 8f2813ed14fff..8b959690a0363 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java
@@ -227,23 +227,25 @@ ClientBuilder authentication(String authPluginClassName, Map aut
     /**
      * Set lookup timeout (default: matches operation timeout)
      *
-     * Lookup operations have a different load pattern to other operations. They can be handled by any broker, are not
-     * proportional to throughput, and are harmless to retry. Given this, it makes sense to allow them to retry longer
-     * than normal operation, especially if they experience a timeout.
+     * 

+     * Lookup operations have a different load pattern to other operations.
+     * They can be handled by any broker, are not proportional to throughput,
+     * and are harmless to retry. Given this, it makes sense to allow them to
+     * retry longer than normal operation, especially if they experience a timeout.
      *
-     * By default this is set to match operation timeout. This is to maintain legacy behaviour. However, in practice
-     * it should be set to 5-10x the operation timeout.
+     * 

+     * By default, this is set to match operation timeout. This is to maintain legacy behaviour.
+     * However, in practice it should be set to 5-10x the operation timeout.
      *
-     * @param lookupTimeout
-     *            lookup timeout
-     * @param unit
-     *            time unit for {@code lookupTimeout}
+     * @param lookupTimeout lookup timeout
+     * @param unit time unit for {@code lookupTimeout}
      * @return the client builder instance
      */
     ClientBuilder lookupTimeout(int lookupTimeout, TimeUnit unit);
 
     /**
-     * Set the number of threads to be used for handling connections to brokers (default: 1 thread).
+     * Set the number of threads to be used for handling connections to brokers
+     * (default: Runtime.getRuntime().availableProcessors()).
      *
      * @param numIoThreads the number of IO threads
      * @return the client builder instance
@@ -251,11 +253,12 @@ ClientBuilder authentication(String authPluginClassName, Map aut
     ClientBuilder ioThreads(int numIoThreads);
 
     /**
-     * Set the number of threads to be used for message listeners (default: 1 thread).
+     * Set the number of threads to be used for message listeners
+     * (default: Runtime.getRuntime().availableProcessors()).
      *
      * 
The listener thread pool is shared across all the consumers and readers that are
-     * using a "listener" model to get messages. For a given consumer, the listener will be
-     * always invoked from the same thread, to ensure ordering.
+     * using a "listener" model to get messages. For a given consumer, the listener will
+     * always be invoked from the same thread, to ensure ordering.
      *
      * @param numListenerThreads the number of listener threads
      * @return the client builder instance

From bbf52736f7542423b9699387eb9521c35e187816 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 29 Mar 2023 13:13:18 +0800
Subject: [PATCH 121/404] [improve][broker][PIP-195] Merge multiple buckets at
 once (#19927)

---
 .../mledger/impl/ManagedCursorImpl.java       |   5 +-
 .../bucket/BucketDelayedDeliveryTracker.java  | 199 ++++++++++--------
 .../CombinedSegmentDelayedIndexQueue.java     | 100 +++++----
 .../BucketDelayedDeliveryTrackerTest.java     |   8 +-
 .../delayed/bucket/DelayedIndexQueueTest.java |  21 +-
 5 files changed, 180 insertions(+), 153 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 5851395b08566..05cad09b01833 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -363,8 +363,7 @@ private CompletableFuture computeCursorProperties(
                 name, copy, lastCursorLedgerStat, new MetaStoreCallback<>() {
                     @Override
                     public void operationComplete(Void result, Stat stat) {
-                        log.info("[{}] Updated ledger cursor: {} properties {}", ledger.getName(),
-                                name, cursorProperties);
+                        log.info("[{}] Updated ledger cursor: {}", ledger.getName(), name);
                         ManagedCursorImpl.this.cursorProperties = Collections.unmodifiableMap(newProperties);
                         updateCursorLedgerStat(copy, stat);
                         updateCursorPropertiesResult.complete(result);
@@ -373,7 +372,7 @@ public void operationComplete(Void result, Stat stat) {
                     @Override
                     public void operationFailed(MetaStoreException e) {
                         log.error("[{}] Error while updating ledger cursor: {} properties {}", ledger.getName(),
-                                name, cursorProperties, e);
+                                name, newProperties, e);
                         updateCursorPropertiesResult.completeExceptionally(e);
                     }
                 });
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 39973928abf5d..ef7be187cec3d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -32,6 +32,7 @@
 import io.netty.util.Timer;
 import java.time.Clock;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -43,6 +44,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 import javax.annotation.concurrent.ThreadSafe;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -93,6 +95,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private static final Long INVALID_BUCKET_ID = -1L;
 
+    private static final int MAX_MERGE_NUM = 4;
+
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
@@ -341,115 +345,136 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
         return true;
     }
 
-    private synchronized CompletableFuture asyncMergeBucketSnapshot() {
-        List values = immutableBuckets.asMapOfRanges().values().stream().toList();
+    private synchronized List selectMergedBuckets(final List values, int mergeNum) {
+        checkArgument(mergeNum < values.size());
         long minNumberMessages = Long.MAX_VALUE;
         long minScheduleTimestamp = Long.MAX_VALUE;
         int minIndex = -1;
-        for (int i = 0; i + 1 < values.size(); i++) {
-            ImmutableBucket bucketL = values.get(i);
-            ImmutableBucket bucketR = values.get(i + 1);
-            // We should skip the bucket which last segment already been load to memory, avoid record replicated index.
-            if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()
-                    && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId()
-                    // Skip the bucket that is merging
-                    && !bucketL.merging && !bucketR.merging){
-                long scheduleTimestamp =
-                        Math.min(bucketL.firstScheduleTimestamps.get(bucketL.currentSegmentEntryId + 1),
-                                bucketR.firstScheduleTimestamps.get(bucketR.currentSegmentEntryId + 1));
-                long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages;
-                if (scheduleTimestamp <= minScheduleTimestamp) {
-                    minScheduleTimestamp = scheduleTimestamp;
-                    if (numberMessages < minNumberMessages) {
-                        minNumberMessages = numberMessages;
+        for (int i = 0; i + (mergeNum - 1) < values.size(); i++) {
+            List immutableBuckets = values.subList(i, i + mergeNum);
+            if (immutableBuckets.stream().allMatch(bucket -> {
+                // We should skip the bucket which last segment already been load to memory,
+                // avoid record replicated index.
+                return bucket.lastSegmentEntryId > bucket.currentSegmentEntryId && !bucket.merging;
+            })) {
+                long numberMessages = immutableBuckets.stream()
+                        .mapToLong(bucket -> bucket.numberBucketDelayedMessages)
+                        .sum();
+                if (numberMessages <= minNumberMessages) {
+                    minNumberMessages = numberMessages;
+                    long scheduleTimestamp = immutableBuckets.stream()
+                            .mapToLong(bucket -> bucket.firstScheduleTimestamps.get(bucket.currentSegmentEntryId + 1))
+                            .min().getAsLong();
+                    if (scheduleTimestamp < minScheduleTimestamp) {
+                        minScheduleTimestamp = scheduleTimestamp;
                         minIndex = i;
                     }
                 }
             }
         }
 
-        if (minIndex == -1) {
-            log.warn("[{}] Can't find able merged bucket", dispatcher.getName());
-            return CompletableFuture.completedFuture(null);
+        if (minIndex >= 0) {
+            return values.subList(minIndex, minIndex + MAX_MERGE_NUM);
+        } else if (mergeNum > 2){
+            return selectMergedBuckets(values, mergeNum - 1);
+        } else {
+            return Collections.emptyList();
         }
+    }
 
-        ImmutableBucket immutableBucketA = values.get(minIndex);
-        ImmutableBucket immutableBucketB = values.get(minIndex + 1);
+    private synchronized CompletableFuture asyncMergeBucketSnapshot() {
+        List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList();
+        List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM);
 
+        if (toBeMergeImmutableBuckets.isEmpty()) {
+            log.warn("[{}] Can't find able merged buckets", dispatcher.getName());
+            return CompletableFuture.completedFuture(null);
+        }
+
+        final String bucketsStr = toBeMergeImmutableBuckets.stream().map(Bucket::bucketKey).collect(
+                Collectors.joining(",")).replaceAll(DELAYED_BUCKET_KEY_PREFIX + "_", "");
         if (log.isDebugEnabled()) {
-            log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(),
-                    immutableBucketA.bucketKey(), immutableBucketB.bucketKey());
+            log.info("[{}] Merging bucket snapshot, bucketKeys: {}", dispatcher.getName(), bucketsStr);
         }
 
-        immutableBucketA.merging = true;
-        immutableBucketB.merging = true;
-        return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> {
+        for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
+            immutableBucket.merging = true;
+        }
+        return asyncMergeBucketSnapshot(toBeMergeImmutableBuckets).whenComplete((__, ex) -> {
             synchronized (this) {
-                immutableBucketA.merging = false;
-                immutableBucketB.merging = false;
+                for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
+                    immutableBucket.merging = false;
+                }
             }
             if (ex != null) {
-                log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}",
-                        dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex);
+                log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}",
+                        dispatcher.getName(), bucketsStr, ex);
             } else {
-                log.info("[{}] Merge bucket snapshot finish, bucketAKey: {}, bucketBKey: {}",
-                        dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey());
+                log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}",
+                        dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size());
             }
         });
     }
 
-    private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA,
-                                                                          ImmutableBucket bucketB) {
-        CompletableFuture createAFuture = bucketA.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
-        CompletableFuture createBFuture = bucketB.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
+    private synchronized CompletableFuture asyncMergeBucketSnapshot(List buckets) {
+        List> createFutures =
+                buckets.stream().map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE))
+                        .toList();
 
-        return CompletableFuture.allOf(createAFuture, createBFuture).thenCompose(bucketId -> {
-            if (INVALID_BUCKET_ID.equals(createAFuture.join()) || INVALID_BUCKET_ID.equals(createBFuture.join())) {
+        return FutureUtil.waitForAll(createFutures).thenCompose(bucketId -> {
+            if (createFutures.stream().anyMatch(future -> INVALID_BUCKET_ID.equals(future.join()))) {
                 return FutureUtil.failedFuture(new RuntimeException("Can't merge buckets due to bucket create failed"));
             }
 
-            CompletableFuture> futureA =
-                    bucketA.getRemainSnapshotSegment();
-            CompletableFuture> futureB =
-                    bucketB.getRemainSnapshotSegment();
-            return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap)
+            List>> getRemainFutures =
+                    buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList();
+
+            return FutureUtil.waitForAll(getRemainFutures)
+                    .thenApply(__ -> {
+                        return CombinedSegmentDelayedIndexQueue.wrap(
+                                getRemainFutures.stream().map(CompletableFuture::join).toList());
+                    })
                     .thenAccept(combinedDelayedIndexQueue -> {
                         synchronized (BucketDelayedDeliveryTracker.this) {
                             Pair immutableBucketDelayedIndexPair =
                                     lastMutableBucket.createImmutableBucketAndAsyncPersistent(
                                             timeStepPerBucketSnapshotSegmentInMillis,
                                             maxIndexesPerBucketSnapshotSegment,
-                                            sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
-                                            bucketB.endLedgerId);
+                                            sharedBucketPriorityQueue, combinedDelayedIndexQueue,
+                                            buckets.get(0).startLedgerId,
+                                            buckets.get(buckets.size() - 1).endLedgerId);
 
                             // Merge bit map to new bucket
-                            Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
-                            Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap();
-                            Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA);
-                            delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> {
-                                delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> {
-                                    if (bitMapA == null) {
-                                        return bitMapB;
-                                    }
-
-                                    bitMapA.or(bitMapB);
-                                    return bitMapA;
+                            Map delayedIndexBitMap =
+                                    new HashMap<>(buckets.get(0).getDelayedIndexBitMap());
+                            for (int i = 1; i < buckets.size(); i++) {
+                                buckets.get(i).delayedIndexBitMap.forEach((ledgerId, bitMapB) -> {
+                                    delayedIndexBitMap.compute(ledgerId, (k, bitMap) -> {
+                                        if (bitMap == null) {
+                                            return bitMapB;
+                                        }
+
+                                        bitMap.or(bitMapB);
+                                        return bitMap;
+                                    });
                                 });
-                            });
+                            }
                             immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap);
 
                             afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
 
                             immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture()
                                     .orElse(NULL_LONG_PROMISE).thenCompose(___ -> {
-                                        CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot();
-                                        CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot();
-                                        return CompletableFuture.allOf(removeAFuture, removeBFuture);
+                                        List> removeFutures =
+                                                buckets.stream().map(ImmutableBucket::asyncDeleteBucketSnapshot)
+                                                        .toList();
+                                        return FutureUtil.waitForAll(removeFutures);
                                     });
 
-                            Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges();
-                            immutableBucketMap.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
-                            immutableBucketMap.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
+                            for (ImmutableBucket bucket : buckets) {
+                                immutableBuckets.asMapOfRanges()
+                                        .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
+                            }
                         }
                     });
         });
@@ -593,14 +618,12 @@ public boolean shouldPauseAllDeliveries() {
 
     @Override
     public synchronized CompletableFuture clear() {
-        return cleanImmutableBuckets(true).thenRun(() -> {
-            synchronized (this) {
-                sharedBucketPriorityQueue.clear();
-                lastMutableBucket.clear();
-                snapshotSegmentLastIndexTable.clear();
-                numberDelayedMessages = 0;
-            }
-        });
+        CompletableFuture future = cleanImmutableBuckets();
+        sharedBucketPriorityQueue.clear();
+        lastMutableBucket.clear();
+        snapshotSegmentLastIndexTable.clear();
+        numberDelayedMessages = 0;
+        return future;
     }
 
     @Override
@@ -609,30 +632,24 @@ public synchronized void close() {
         lastMutableBucket.close();
         sharedBucketPriorityQueue.close();
         try {
-            cleanImmutableBuckets(false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+            List> completableFutures = immutableBuckets.asMapOfRanges().values().stream()
+                    .map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE)).toList();
+            FutureUtil.waitForAll(completableFutures).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
         } catch (Exception e) {
             log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e);
         }
     }
 
-    private CompletableFuture cleanImmutableBuckets(boolean delete) {
-        if (immutableBuckets != null) {
-            List> futures = new ArrayList<>();
-            Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
-            while (iterator.hasNext()) {
-                ImmutableBucket bucket = iterator.next();
-                if (delete) {
-                    futures.add(bucket.clear());
-                } else {
-                    bucket.getSnapshotCreateFuture().ifPresent(future -> futures.add(future.thenApply(x -> null)));
-                }
-                numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
-                iterator.remove();
-            }
-            return FutureUtil.waitForAll(futures);
-        } else {
-            return CompletableFuture.completedFuture(null);
+    private CompletableFuture cleanImmutableBuckets() {
+        List> futures = new ArrayList<>();
+        Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
+        while (iterator.hasNext()) {
+            ImmutableBucket bucket = iterator.next();
+            futures.add(bucket.clear());
+            numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
+            iterator.remove();
         }
+        return FutureUtil.waitForAll(futures);
     }
 
     private boolean removeIndexBit(long ledgerId, long entryId) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
index 3f89cc9fdfb15..5655a26878296 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
@@ -18,37 +18,48 @@
  */
 package org.apache.pulsar.broker.delayed.bucket;
 
+import java.util.Comparator;
 import java.util.List;
+import java.util.Objects;
+import java.util.PriorityQueue;
 import javax.annotation.concurrent.NotThreadSafe;
+import lombok.AllArgsConstructor;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
 
 @NotThreadSafe
 class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue {
 
-    private final List segmentListA;
-    private final List segmentListB;
+    @AllArgsConstructor
+    static class Node {
+        List segmentList;
 
-    private int segmentListACursor = 0;
-    private int segmentListBCursor = 0;
-    private int segmentACursor = 0;
-    private int segmentBCursor = 0;
+        int segmentListCursor;
 
-    private CombinedSegmentDelayedIndexQueue(List segmentListA,
-                                             List segmentListB) {
-        this.segmentListA = segmentListA;
-        this.segmentListB = segmentListB;
+        int segmentCursor;
     }
 
-    public static CombinedSegmentDelayedIndexQueue wrap(
-            List segmentListA,
-            List segmentListB) {
-        return new CombinedSegmentDelayedIndexQueue(segmentListA, segmentListB);
+    private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare(
+            node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor),
+            node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor));
+
+    private final PriorityQueue kpq;
+
+    private CombinedSegmentDelayedIndexQueue(List> segmentLists) {
+        this.kpq = new PriorityQueue<>(segmentLists.size(), COMPARATOR_NODE);
+        for (List segmentList : segmentLists) {
+            Node node = new Node(segmentList, 0, 0);
+            kpq.offer(node);
+        }
+    }
+
+    public static CombinedSegmentDelayedIndexQueue wrap(List> segmentLists) {
+        return new CombinedSegmentDelayedIndexQueue(segmentLists);
     }
 
     @Override
     public boolean isEmpty() {
-        return segmentListACursor >= segmentListA.size() && segmentListBCursor >= segmentListB.size();
+        return kpq.isEmpty();
     }
 
     @Override
@@ -62,48 +73,35 @@ public DelayedIndex pop() {
     }
 
     private DelayedIndex getValue(boolean needAdvanceCursor) {
-        // skip empty segment
-        while (segmentListACursor < segmentListA.size()
-                && segmentListA.get(segmentListACursor).getIndexesCount() == 0) {
-            segmentListACursor++;
-        }
-        while (segmentListBCursor < segmentListB.size()
-                && segmentListB.get(segmentListBCursor).getIndexesCount() == 0) {
-            segmentListBCursor++;
-        }
+        Node node = kpq.peek();
+        Objects.requireNonNull(node);
 
-        DelayedIndex delayedIndexA = null;
-        DelayedIndex delayedIndexB = null;
-        if (segmentListACursor >= segmentListA.size()) {
-            delayedIndexB = segmentListB.get(segmentListBCursor).getIndexes(segmentBCursor);
-        } else if (segmentListBCursor >= segmentListB.size()) {
-            delayedIndexA = segmentListA.get(segmentListACursor).getIndexes(segmentACursor);
-        } else {
-            delayedIndexA = segmentListA.get(segmentListACursor).getIndexes(segmentACursor);
-            delayedIndexB = segmentListB.get(segmentListBCursor).getIndexes(segmentBCursor);
+        SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor);
+        DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor);
+        if (!needAdvanceCursor) {
+            return delayedIndex;
         }
 
-        DelayedIndex resultValue;
-        if (delayedIndexB == null || (delayedIndexA != null && COMPARATOR.compare(delayedIndexA, delayedIndexB) < 0)) {
-            resultValue = delayedIndexA;
-            if (needAdvanceCursor) {
-                if (++segmentACursor >= segmentListA.get(segmentListACursor).getIndexesCount()) {
-                    segmentListA.set(segmentListACursor, null);
-                    ++segmentListACursor;
-                    segmentACursor = 0;
-                }
-            }
-        } else {
-            resultValue = delayedIndexB;
-            if (needAdvanceCursor) {
-                if (++segmentBCursor >= segmentListB.get(segmentListBCursor).getIndexesCount()) {
-                    segmentListB.set(segmentListBCursor, null);
-                    ++segmentListBCursor;
-                    segmentBCursor = 0;
+        kpq.poll();
+
+        if (node.segmentCursor + 1 < snapshotSegment.getIndexesCount()) {
+            node.segmentCursor++;
+            kpq.offer(node);
+        } else  {
+            // help GC
+            node.segmentList.set(node.segmentListCursor, null);
+            while (node.segmentListCursor + 1 < node.segmentList.size()) {
+                node.segmentListCursor++;
+                node.segmentCursor = 0;
+
+                // skip empty segment
+                if (node.segmentList.get(node.segmentListCursor).getIndexesCount() > 0) {
+                    kpq.offer(node);
+                    break;
                 }
             }
         }
 
-        return resultValue;
+        return delayedIndex;
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index fbb866d48ed69..717bada7705dc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -267,9 +267,7 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) {
 
         int size = tracker.getImmutableBuckets().asMapOfRanges().size();
 
-        Awaitility.await().untilAsserted(() -> {
-            assertEquals(10, size);
-        });
+        assertTrue(size <= 10);
 
         tracker.addMessage(111, 1011, 111 * 10);
         Awaitility.await().untilAsserted(() -> {
@@ -333,9 +331,7 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) {
 
         int size = tracker.getImmutableBuckets().asMapOfRanges().size();
 
-        Awaitility.await().untilAsserted(() -> {
-            assertEquals(10, size);
-        });
+        assertTrue(size <= 10);
 
         tracker.addMessage(111, 1011, 111 * 10);
         Awaitility.await().untilAsserted(() -> {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
index 865ccb6934ab7..8f87f0d49a20c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
@@ -101,8 +101,25 @@ public void testCombinedSegmentDelayedIndexQueue() {
         segmentListB.add(snapshotSegmentB);
         segmentListB.add(SnapshotSegment.newBuilder().build());
 
+        List listC = new ArrayList<>();
+        for (int i = 10; i < 30; i+=2) {
+            DelayedIndex delayedIndex =
+                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L)
+                            .build();
+
+            DelayedIndex delayedIndex2 =
+                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L)
+                            .build();
+            listC.add(delayedIndex);
+            listC.add(delayedIndex2);
+        }
+
+        SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build();
+        List segmentListC = new ArrayList<>();
+        segmentListC.add(snapshotSegmentC);
+
         CombinedSegmentDelayedIndexQueue delayedIndexQueue =
-                CombinedSegmentDelayedIndexQueue.wrap(segmentListA, segmentListB);
+                CombinedSegmentDelayedIndexQueue.wrap(List.of(segmentListA, segmentListB, segmentListC));
 
         int count = 0;
         while (!delayedIndexQueue.isEmpty()) {
@@ -114,7 +131,7 @@ public void testCombinedSegmentDelayedIndexQueue() {
                 Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0);
             }
         }
-        Assert.assertEquals(38, count);
+        Assert.assertEquals(58, count);
     }
 
     @Test

From 7cb48fd9d41f8606d86a0503b0130aec6b9d00d5 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com>
Date: Tue, 28 Mar 2023 23:54:05 -0700
Subject: [PATCH 122/404] [improve][io] KCA: option to collapse partitioned
 topics (#19923)

---
 pulsar-io/kafka-connect-adaptor/pom.xml       |  7 ++
 .../io/kafka/connect/KafkaConnectSink.java    | 20 ++++-
 .../connect/PulsarKafkaConnectSinkConfig.java |  5 ++
 .../kafka/connect/KafkaConnectSinkTest.java   | 88 +++++++++++++++++++
 4 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml
index 5e192be95ecf4..12ebda087eb3d 100644
--- a/pulsar-io/kafka-connect-adaptor/pom.xml
+++ b/pulsar-io/kafka-connect-adaptor/pom.xml
@@ -39,6 +39,13 @@
       provided
     
 
+    
+      ${project.groupId}
+      pulsar-common
+      ${project.version}
+      compile
+    
+
     
       com.fasterxml.jackson.core
       jackson-databind
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index 06f66f60380d9..475724cc4e545 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.schema.GenericObject;
 import org.apache.pulsar.client.api.schema.KeyValueSchema;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.schema.KeyValue;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.functions.api.Record;
@@ -91,6 +92,9 @@ public class KafkaConnectSink implements Sink {
     protected String topicName;
 
     private boolean sanitizeTopicName = false;
+    // Thi is a workaround for https://github.com/apache/pulsar/issues/19922
+    private boolean collapsePartitionedTopics = false;
+
     private final Cache sanitizedTopicCache =
             CacheBuilder.newBuilder().maximumSize(1000)
                     .expireAfterAccess(30, TimeUnit.MINUTES).build();
@@ -159,6 +163,7 @@ public void open(Map config, SinkContext ctx) throws Exception {
         topicName = kafkaSinkConfig.getTopic();
         unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable();
         sanitizeTopicName = kafkaSinkConfig.isSanitizeTopicName();
+        collapsePartitionedTopics = kafkaSinkConfig.isCollapsePartitionedTopics();
 
         useIndexAsOffset = kafkaSinkConfig.isUseIndexAsOffset();
         maxBatchBitsForOffset = kafkaSinkConfig.getMaxBatchBitsForOffset();
@@ -417,8 +422,19 @@ static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId me
 
     @SuppressWarnings("rawtypes")
     protected SinkRecord toSinkRecord(Record sourceRecord) {
-        final int partition = sourceRecord.getPartitionIndex().orElse(0);
-        final String topic = sanitizeNameIfNeeded(sourceRecord.getTopicName().orElse(topicName), sanitizeTopicName);
+        final int partition;
+        final String topic;
+
+        if (collapsePartitionedTopics
+                && sourceRecord.getTopicName().isPresent()
+                && TopicName.get(sourceRecord.getTopicName().get()).isPartitioned()) {
+            TopicName tn = TopicName.get(sourceRecord.getTopicName().get());
+            partition = tn.getPartitionIndex();
+            topic = sanitizeNameIfNeeded(tn.getPartitionedTopicName(), sanitizeTopicName);
+        } else {
+            partition = sourceRecord.getPartitionIndex().orElse(0);
+            topic = sanitizeNameIfNeeded(sourceRecord.getTopicName().orElse(topicName), sanitizeTopicName);
+        }
         final Object key;
         final Object value;
         final Schema keySchema;
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
index 19dd784578915..2525081a41ebb 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
@@ -94,6 +94,11 @@ public class PulsarKafkaConnectSinkConfig implements Serializable {
                     + "In some cases it may result in topic name collisions (topic_a and topic.a will become the same)")
     private boolean sanitizeTopicName = false;
 
+    @FieldDoc(
+            defaultValue = "false",
+            help = "Supply kafka record with topic name without -partition- suffix for partitioned topics.")
+    private boolean collapsePartitionedTopics = false;
+
     public static PulsarKafkaConnectSinkConfig load(String yamlFile) throws IOException {
         ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
         return mapper.readValue(new File(yamlFile), PulsarKafkaConnectSinkConfig.class);
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index e9d454ed2fd5a..6ccfa3b71a2f7 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -1571,6 +1571,94 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception {
         assertEquals(ref.getBatchIdx(), batchIdx);
     }
 
+    @Test
+    public void collapsePartitionedTopicEnabledTest() throws Exception {
+        testCollapsePartitionedTopic(true,
+                "persistent://a/b/fake-topic-partition-0",
+                "persistent://a/b/fake-topic",
+                0);
+
+        testCollapsePartitionedTopic(true,
+                "persistent://a/b/fake-topic-partition-1",
+                "persistent://a/b/fake-topic",
+                1);
+
+        testCollapsePartitionedTopic(true,
+                "persistent://a/b/fake-topic",
+                "persistent://a/b/fake-topic",
+                0);
+
+        testCollapsePartitionedTopic(true,
+                "fake-topic-partition-5",
+                "persistent://public/default/fake-topic",
+                5);
+    }
+
+    @Test
+    public void collapsePartitionedTopicDisabledTest() throws Exception {
+        testCollapsePartitionedTopic(false,
+                "persistent://a/b/fake-topic-partition-0",
+                "persistent://a/b/fake-topic-partition-0",
+                0);
+
+        testCollapsePartitionedTopic(false,
+                "persistent://a/b/fake-topic-partition-1",
+                "persistent://a/b/fake-topic-partition-1",
+                0);
+
+        testCollapsePartitionedTopic(false,
+                "persistent://a/b/fake-topic",
+                "persistent://a/b/fake-topic",
+                0);
+
+        testCollapsePartitionedTopic(false,
+                "fake-topic-partition-5",
+                "fake-topic-partition-5",
+                0);
+    }
+
+    private void testCollapsePartitionedTopic(boolean isEnabled,
+                                              String pulsarTopic,
+                                              String expectedKafkaTopic,
+                                              int expectedPartition) throws Exception {
+        props.put("kafkaConnectorSinkClass", SchemaedFileStreamSinkConnector.class.getCanonicalName());
+        props.put("collapsePartitionedTopics", Boolean.toString(isEnabled));
+
+        KafkaConnectSink sink = new KafkaConnectSink();
+            sink.open(props, context);
+
+        AvroSchema pulsarAvroSchema
+                = AvroSchema.of(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class);
+
+        final GenericData.Record obj = new GenericData.Record(pulsarAvroSchema.getAvroSchema());
+        obj.put("field1", (byte) 10);
+        obj.put("field2", "test");
+        obj.put("field3", (short) 100);
+
+        final GenericRecord rec = getGenericRecord(obj, pulsarAvroSchema);
+        Message msg = mock(MessageImpl.class);
+        when(msg.getValue()).thenReturn(rec);
+        when(msg.getKey()).thenReturn("key");
+        when(msg.hasKey()).thenReturn(true);
+        when(msg.getMessageId()).thenReturn(new MessageIdImpl(1, 0, 0));
+
+        final AtomicInteger status = new AtomicInteger(0);
+        Record record = PulsarRecord.builder()
+                .topicName(pulsarTopic)
+                .message(msg)
+                .schema(pulsarAvroSchema)
+                .ackFunction(status::incrementAndGet)
+                .failFunction(status::decrementAndGet)
+                .build();
+
+        SinkRecord sinkRecord = sink.toSinkRecord(record);
+
+        Assert.assertEquals(sinkRecord.topic(), expectedKafkaTopic);
+        Assert.assertEquals(sinkRecord.kafkaPartition(), expectedPartition);
+
+        sink.close();
+    }
+
     @SneakyThrows
     private java.util.Date getDateFromString(String dateInString) {
         SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss");

From ee5ac8607ebd7f90843789d4eead48de8600c1f0 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Wed, 29 Mar 2023 02:30:24 -0700
Subject: [PATCH 123/404] [improve][broker] PIP-192 Improved Auto Unload Logic
 (#19813)

Master Issue: Master Issue: https://github.com/apache/pulsar/issues/16691, https://github.com/apache/pulsar/issues/18099

### Motivation

Raising a PR to implement Master Issue: https://github.com/apache/pulsar/issues/16691, https://github.com/apache/pulsar/issues/18099

We want to reduce unload frequencies from flaky traffic.

### Modifications
This PR
- Introduced a config `loadBalancerSheddingConditionHitCountThreshold` to further restrict shedding conditions based on the hit count.
- Normalized offload traffic
- Lowered the default `loadBalanceSheddingDelayInSeconds` value from 600 to 180, as 10 mins are too long. 3 mins can be long enough to catch the new load after unloads.
- Changed the config `loadBalancerBundleLoadReportPercentage` to `loadBalancerMaxNumberOfBundlesInBundleLoadReport` to make the topk bundle count absolute instead of relative.
- Renamed `loadBalancerNamespaceBundleSplitConditionThreshold` to `loadBalancerNamespaceBundleSplitConditionHitCountThreshold` to be consistent with `*ConditionHitCountThreshold`.
- Renamed `loadBalancerMaxNumberOfBrokerTransfersPerCycle ` to `loadBalancerMaxNumberOfBrokerSheddingPerCycle`.
- Added LoadDataStore cleanup logic in BSC monitor.
- Added `msgThroughputEMA` in BrokerLoadData to smooth the broker throughput info.
- Updated Topk bundles sorted in a ascending order (instead of descending)
- Update some info logs to only show in the debug mode.
- Added load data tombstone upon Own, Releasing, Splitting
- Added the bundle ownership(isOwned) check upon split and unload.
- Added swap unload logic
---
 conf/broker.conf                              |  85 +++
 .../pulsar/broker/ServiceConfiguration.java   |  23 +-
 .../extensions/ExtensibleLoadManagerImpl.java |  67 ++-
 .../channel/ServiceUnitStateChannel.java      |  25 +
 .../channel/ServiceUnitStateChannelImpl.java  |  85 ++-
 .../extensions/data/BrokerLoadData.java       |  34 +-
 .../extensions/models/TopKBundles.java        |   2 +-
 .../extensions/models/UnloadCounter.java      |   4 +-
 .../extensions/models/UnloadDecision.java     |   2 +-
 .../reporter/BrokerLoadDataReporter.java      |  90 ++-
 .../reporter/TopBundleLoadDataReporter.java   |  63 ++-
 .../scheduler/NamespaceUnloadStrategy.java    |   8 +
 .../extensions/scheduler/SplitScheduler.java  |  16 +-
 .../extensions/scheduler/TransferShedder.java | 469 +++++++++++-----
 .../extensions/scheduler/UnloadScheduler.java |  19 +-
 ...faultNamespaceBundleSplitStrategyImpl.java | 144 +++--
 .../LeastResourceUsageWithWeight.java         |   8 +-
 .../ExtensibleLoadManagerImplTest.java        |   6 +-
 .../channel/ServiceUnitStateChannelTest.java  |  66 +++
 .../extensions/data/BrokerLoadDataTest.java   |  10 +-
 .../extensions/models/TopKBundlesTest.java    |  14 +-
 .../reporter/BrokerLoadDataReporterTest.java  |  93 +++-
 .../TopBundleLoadDataReporterTest.java        |  90 ++-
 .../scheduler/TransferShedderTest.java        | 515 +++++++++++++++---
 .../scheduler/UnloadSchedulerTest.java        |  12 +-
 ...faultNamespaceBundleSplitStrategyTest.java |  49 +-
 .../loadbalancer/NamespaceBundleStats.java    |   2 +
 .../data/loadbalancer/ResourceUsage.java      |   2 +
 28 files changed, 1648 insertions(+), 355 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index d52adb254563d..1bbb12313f9a6 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1373,6 +1373,91 @@ loadBalancerBundleUnloadMinThroughputThreshold=10
 # Time to wait for the unloading of a namespace bundle
 namespaceBundleUnloadingTimeoutMs=60000
 
+### --- Load balancer extension --- ###
+
+# Option to enable the debug mode for the load balancer logics.
+# The debug mode prints more logs to provide more information such as load balance states and decisions.
+# (only used in load balancer extension logics)
+loadBalancerDebugModeEnabled=false
+
+# The target standard deviation of the resource usage across brokers
+# (100% resource usage is 1.0 load).
+# The shedder logic tries to distribute bundle load across brokers to meet this target std.
+# The smaller value will incur load balancing more frequently.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerBrokerLoadTargetStd=0.25
+
+# Threshold to the consecutive count of fulfilled shedding(unload) conditions.
+# If the unload scheduler consecutively finds bundles that meet unload conditions
+# many times bigger than this threshold, the scheduler will shed the bundles.
+# The bigger value will incur less bundle unloading/transfers.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerSheddingConditionHitCountThreshold=3
+
+# Option to enable the bundle transfer mode when distributing bundle loads.
+# On: transfer bundles from overloaded brokers to underloaded
+# -- pre-assigns the destination broker upon unloading).
+# Off: unload bundles from overloaded brokers
+# -- post-assigns the destination broker upon lookups).
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerTransferEnabled=true
+
+# Maximum number of brokers to unload bundle load for each unloading cycle.
+# The bigger value will incur more unloading/transfers for each unloading cycle.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerMaxNumberOfBrokerSheddingPerCycle=3
+
+# Delay (in seconds) to the next unloading cycle after unloading.
+# The logic tries to give enough time for brokers to recompute load after unloading.
+# The bigger value will delay the next unloading cycle longer.
+# (only used in load balancer extension TransferSheddeer)
+loadBalanceSheddingDelayInSeconds=180
+
+# Broker load data time to live (TTL in seconds).
+# The logic tries to avoid (possibly unavailable) brokers with out-dated load data,
+# and those brokers will be ignored in the load computation.
+# When tuning this value, please consider loadBalancerReportUpdateMaxIntervalMinutes.
+#The current default is loadBalancerReportUpdateMaxIntervalMinutes * 2.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerBrokerLoadDataTTLInSeconds=1800
+
+# Max number of bundles in bundle load report from each broker.
+# The load balancer distributes bundles across brokers,
+# based on topK bundle load data and other broker load data.
+# The bigger value will increase the overhead of reporting many bundles in load data.
+# (only used in load balancer extension logics)
+loadBalancerMaxNumberOfBundlesInBundleLoadReport=10
+
+# Service units'(bundles) split interval. Broker periodically checks whether
+# some service units(e.g. bundles) should split if they become hot-spots.
+# (only used in load balancer extension logics)
+loadBalancerSplitIntervalMinutes=1
+
+# Max number of bundles to split to per cycle.
+# (only used in load balancer extension logics)
+loadBalancerMaxNumberOfBundlesToSplitPerCycle=10
+
+# Threshold to the consecutive count of fulfilled split conditions.
+# If the split scheduler consecutively finds bundles that meet split conditions
+# many times bigger than this threshold, the scheduler will trigger splits on the bundles
+# (if the number of bundles is less than loadBalancerNamespaceMaximumBundles).
+# (only used in load balancer extension logics)
+loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3
+
+# After this delay, the service-unit state channel tombstones any service units (e.g., bundles)
+# in semi-terminal states. For example, after splits, parent bundles will be `deleted`,
+# and then after this delay, the parent bundles' state will be `tombstoned`
+# in the service-unit state channel.
+# Pulsar does not immediately remove such semi-terminal states
+# to avoid unnecessary system confusion,
+# as the bundles in the `tombstoned` state might temporarily look available to reassign.
+# Rarely, one could lower this delay in order to aggressively clean
+# the service-unit state channel when there are a large number of bundles.
+# minimum value = 30 secs
+# (only used in load balancer extension logics)
+loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds=604800
+
+
 ### --- Replication --- ###
 
 # Enable replication metrics
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index e2bcf03353b59..03e0fec0a5346 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2506,6 +2506,17 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     )
     private double loadBalancerBrokerLoadTargetStd = 0.25;
 
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Threshold to the consecutive count of fulfilled shedding(unload) conditions. "
+                    + "If the unload scheduler consecutively finds bundles that meet unload conditions "
+                    + "many times bigger than this threshold, the scheduler will shed the bundles. "
+                    + "The bigger value will incur less bundle unloading/transfers. "
+                    + "(only used in load balancer extension TransferSheddeer)"
+    )
+    private int loadBalancerSheddingConditionHitCountThreshold = 3;
+
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
             dynamic = true,
@@ -2521,11 +2532,11 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
             dynamic = true,
-            doc = "Maximum number of brokers to transfer bundle load for each unloading cycle. "
+            doc = "Maximum number of brokers to unload bundle load for each unloading cycle. "
                     + "The bigger value will incur more unloading/transfers for each unloading cycle. "
                     + "(only used in load balancer extension TransferSheddeer)"
     )
-    private int loadBalancerMaxNumberOfBrokerTransfersPerCycle = 3;
+    private int loadBalancerMaxNumberOfBrokerSheddingPerCycle = 3;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
@@ -2535,7 +2546,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "The bigger value will delay the next unloading cycle longer. "
                     + "(only used in load balancer extension TransferSheddeer)"
     )
-    private long loadBalanceUnloadDelayInSeconds = 600;
+    private long loadBalanceSheddingDelayInSeconds = 180;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
@@ -2552,13 +2563,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     @FieldContext(
             dynamic = true,
             category = CATEGORY_LOAD_BALANCER,
-            doc = "Percentage of bundles to compute topK bundle load data from each broker. "
+            doc = "Max number of bundles in bundle load report from each broker. "
                     + "The load balancer distributes bundles across brokers, "
                     + "based on topK bundle load data and other broker load data."
                     + "The bigger value will increase the overhead of reporting many bundles in load data. "
                     + "(only used in load balancer extension logics)"
     )
-    private double loadBalancerBundleLoadReportPercentage = 10;
+    private int loadBalancerMaxNumberOfBundlesInBundleLoadReport = 10;
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
             doc = "Service units'(bundles) split interval. Broker periodically checks whether "
@@ -2582,7 +2593,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "(if the number of bundles is less than loadBalancerNamespaceMaximumBundles). "
                     + "(only used in load balancer extension logics)"
     )
-    private int loadBalancerNamespaceBundleSplitConditionThreshold = 5;
+    private int loadBalancerNamespaceBundleSplitConditionHitCountThreshold = 3;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 7aefef596e7cb..3e078d0a5eb20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -85,6 +85,7 @@
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
+import org.slf4j.Logger;
 
 @Slf4j
 public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
@@ -101,6 +102,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private static final long MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS = 200;
 
+    private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000;
+
     private PulsarService pulsar;
 
     private ServiceConfiguration conf;
@@ -108,10 +111,12 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
     @Getter
     private BrokerRegistry brokerRegistry;
 
+    @Getter
     private ServiceUnitStateChannel serviceUnitStateChannel;
 
     private AntiAffinityGroupPolicyFilter antiAffinityGroupPolicyFilter;
 
+    @Getter
     private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
     private LoadDataStore brokerLoadDataStore;
@@ -139,6 +144,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private ScheduledFuture brokerLoadDataReportTask;
     private ScheduledFuture topBundlesLoadDataReportTask;
+
+    private ScheduledFuture monitorTask;
     private SplitScheduler splitScheduler;
 
     private UnloadManager unloadManager;
@@ -148,6 +155,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
     private boolean started = false;
 
     private final AssignCounter assignCounter = new AssignCounter();
+    @Getter
     private final UnloadCounter unloadCounter = new UnloadCounter();
     private final SplitCounter splitCounter = new SplitCounter();
 
@@ -162,14 +170,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
             lookupRequests = ConcurrentOpenHashMap.>>newBuilder()
             .build();
-    private final CountDownLatch loadStoreInitWaiter = new CountDownLatch(1);
+    private final CountDownLatch initWaiter = new CountDownLatch(1);
 
     public enum Role {
         Leader,
         Follower
     }
 
-    private Role role;
+    private volatile Role role;
 
     /**
      * Life cycle: Constructor -> initialize -> start -> close.
@@ -194,6 +202,10 @@ public static ExtensibleLoadManagerImpl get(LoadManager loadManager) {
         return loadManagerWrapper.get();
     }
 
+    public static boolean debug(ServiceConfiguration config, Logger log) {
+        return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+    }
+
     @Override
     public void start() throws PulsarServerException {
         if (this.started) {
@@ -231,7 +243,6 @@ public void start() throws PulsarServerException {
             this.brokerLoadDataStore.startTableView();
             this.topBundlesLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class);
-            this.loadStoreInitWaiter.countDown();
         } catch (LoadDataStoreException e) {
             throw new PulsarServerException(e);
         }
@@ -247,7 +258,8 @@ public void start() throws PulsarServerException {
 
         this.topBundleLoadDataReporter =
                 new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore);
-
+        this.serviceUnitStateChannel.listen(brokerLoadDataReporter);
+        this.serviceUnitStateChannel.listen(topBundleLoadDataReporter);
         var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis();
         this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor()
                 .scheduleAtFixedRate(() -> {
@@ -273,13 +285,21 @@ public void start() throws PulsarServerException {
                         interval,
                         interval, TimeUnit.MILLISECONDS);
 
+        this.monitorTask = this.pulsar.getLoadManagerExecutor()
+                .scheduleAtFixedRate(() -> {
+                            monitor();
+                        },
+                        MONITOR_INTERVAL_IN_MILLIS,
+                        MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
+
         this.unloadScheduler = new UnloadScheduler(
-                pulsar, pulsar.getLoadManagerExecutor(), unloadManager,
-                context, serviceUnitStateChannel, antiAffinityGroupPolicyHelper, unloadCounter, unloadMetrics);
+                pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel,
+                unloadCounter, unloadMetrics);
         this.unloadScheduler.start();
         this.splitScheduler = new SplitScheduler(
                 pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
         this.splitScheduler.start();
+        this.initWaiter.countDown();
         this.started = true;
     }
 
@@ -327,7 +347,7 @@ public CompletableFuture> assign(Optional {
                 if (broker.isEmpty()) {
                     String errorMsg = String.format(
-                            "Failed to look up a broker registry:%s for bundle:%s", broker, bundle);
+                            "Failed to get or assign the owner for bundle:%s", bundle);
                     log.error(errorMsg);
                     throw new IllegalStateException(errorMsg);
                 }
@@ -498,6 +518,10 @@ public void close() throws PulsarServerException {
                 topBundlesLoadDataReportTask.cancel(true);
             }
 
+            if (monitorTask != null) {
+                monitorTask.cancel(true);
+            }
+
             this.brokerLoadDataStore.close();
             this.topBundlesLoadDataStore.close();
             this.unloadScheduler.close();
@@ -539,8 +563,8 @@ void playLeader() {
             int retry = 0;
             while (true) {
                 try {
+                    initWaiter.await();
                     serviceUnitStateChannel.scheduleOwnershipMonitor();
-                    loadStoreInitWaiter.await();
                     topBundlesLoadDataStore.startTableView();
                     unloadScheduler.start();
                     break;
@@ -575,8 +599,8 @@ void playFollower() {
             int retry = 0;
             while (true) {
                 try {
+                    initWaiter.await();
                     serviceUnitStateChannel.cancelOwnershipMonitor();
-                    loadStoreInitWaiter.await();
                     topBundlesLoadDataStore.closeTableView();
                     unloadScheduler.close();
                     break;
@@ -631,4 +655,29 @@ public List getMetrics() {
 
         return metricsCollection;
     }
+
+    private void monitor() {
+        try {
+            initWaiter.await();
+
+            // Monitor role
+            // Periodically check the role in case ZK watcher fails.
+            var isChannelOwner = serviceUnitStateChannel.isChannelOwner();
+            if (isChannelOwner) {
+                if (role != Leader) {
+                    log.warn("Current role:{} does not match with the channel ownership:{}. "
+                            + "Playing the leader role.", role, isChannelOwner);
+                    playLeader();
+                }
+            } else {
+                if (role != Follower) {
+                    log.warn("Current role:{} does not match with the channel ownership:{}. "
+                            + "Playing the follower role.", role, isChannelOwner);
+                    playFollower();
+                }
+            }
+        } catch (Throwable e) {
+            log.error("Failed to get the channel ownership.", e);
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index 719c72a67b4ea..4782a31fe0c56 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -64,6 +64,12 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     CompletableFuture isChannelOwnerAsync();
 
+    /**
+     * Checks if the current broker is the owner broker of the system topic in this channel.
+     * @return True if the current broker is the owner. Otherwise, false.
+     */
+    boolean isChannelOwner();
+
     /**
      * Handles the metadata session events to track
      * if the connection between the broker and metadata store is stable or not.
@@ -125,6 +131,25 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     CompletableFuture> getOwnerAsync(String serviceUnit);
 
+    /**
+     * Checks if the target broker is the owner of the service unit.
+     *
+     *
+     * @param serviceUnit (e.g. bundle)
+     * @param targetBroker
+     * @return true if the target broker is the owner. false if unknown.
+     */
+    boolean isOwner(String serviceUnit, String targetBroker);
+
+    /**
+     * Checks if the current broker is the owner of the service unit.
+     *
+     *
+     * @param serviceUnit (e.g. bundle))
+     * @return true if the current broker is the owner. false if unknown.
+     */
+    boolean isOwner(String serviceUnit);
+
     /**
      * Asynchronously publishes the service unit assignment event to the system topic in this channel.
      * It de-duplicates assignment events if there is any ongoing assignment event for the same service unit.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index ca9e55923e799..e7fb59450b25c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
@@ -233,7 +234,7 @@ public void scheduleOwnershipMonitor() {
                                     log.info("Failed to monitor the ownerships. will retry..", e);
                                 }
                             },
-                            ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS);
+                            0, ownershipMonitorDelayTimeInSecs, SECONDS);
             log.info("This leader broker:{} started the ownership monitor.",
                     lookupServiceAddress);
         }
@@ -390,7 +391,7 @@ private boolean validateChannelState(ChannelState targetState, boolean checkLowe
     }
 
     private boolean debug() {
-        return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+        return ExtensibleLoadManagerImpl.debug(config, log);
     }
 
     public CompletableFuture> getChannelOwnerAsync() {
@@ -426,7 +427,7 @@ public CompletableFuture isChannelOwnerAsync() {
         });
     }
 
-    private boolean isChannelOwner() {
+    public boolean isChannelOwner() {
         try {
             return isChannelOwnerAsync().get(
                     MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS);
@@ -437,6 +438,25 @@ private boolean isChannelOwner() {
         }
     }
 
+    public boolean isOwner(String serviceUnit, String targetBroker) {
+        if (!validateChannelState(Started, true)) {
+            throw new IllegalStateException("Invalid channel state:" + channelState.name());
+        }
+        var ownerFuture = getOwnerAsync(serviceUnit);
+        if (!ownerFuture.isDone() || ownerFuture.isCompletedExceptionally() || ownerFuture.isCancelled()) {
+            return false;
+        }
+        var owner = ownerFuture.join();
+        if (owner.isPresent() && StringUtils.equals(targetBroker, owner.get())) {
+            return true;
+        }
+        return false;
+    }
+
+    public boolean isOwner(String serviceUnit) {
+        return isOwner(serviceUnit, lookupServiceAddress);
+    }
+
     public CompletableFuture> getOwnerAsync(String serviceUnit) {
         if (!validateChannelState(Started, true)) {
             return CompletableFuture.failedFuture(
@@ -951,9 +971,11 @@ private void handleBrokerCreationEvent(String broker) {
                             + " Active cleanup job count:{}",
                     broker, cleanupJobs.size());
         } else {
-            log.info("Failed to cancel the ownership cleanup for broker:{}."
-                            + " There was no scheduled cleanup job. Active cleanup job count:{}",
-                    broker, cleanupJobs.size());
+            if (debug()) {
+                log.info("No needs to cancel the ownership cleanup for broker:{}."
+                                + " There was no scheduled cleanup job. Active cleanup job count:{}",
+                        broker, cleanupJobs.size());
+            }
         }
     }
 
@@ -989,6 +1011,8 @@ private void scheduleCleanup(String broker, long delayInSecs) {
                                     log.error("Failed to run the cleanup job for the broker {}, "
                                                     + "totalCleanupErrorCnt:{}.",
                                             broker, totalCleanupErrorCnt.incrementAndGet(), e);
+                                } finally {
+                                    cleanupJobs.remove(broker);
                                 }
                             }
                             , delayed);
@@ -1057,7 +1081,11 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce
 
         double cleanupTime = TimeUnit.NANOSECONDS
                 .toMillis((System.nanoTime() - startTime));
-        // TODO: clean load data stores
+
+        // clean load data stores
+        getContext().topBundleLoadDataStore().removeAsync(broker);
+        getContext().brokerLoadDataStore().removeAsync(broker);
+
         log.info("Completed a cleanup for the inactive broker:{} in {} ms. "
                         + "Cleaned up orphan service units: orphanServiceUnitCleanupCnt:{}, "
                         + "approximate cleanupErrorCnt:{}, metrics:{} ",
@@ -1066,7 +1094,7 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce
                 orphanServiceUnitCleanupCnt,
                 totalCleanupErrorCntStart - totalCleanupErrorCnt.get(),
                 printCleanupMetrics());
-        cleanupJobs.remove(broker);
+
     }
 
     private Optional getRollForwardStateData(
@@ -1123,7 +1151,11 @@ protected void monitorOwnerships(List brokers) {
             return;
         }
 
-        log.info("Started the ownership monitor run for activeBrokerCount:{}", brokers.size());
+        var debug = debug();
+        if (debug) {
+            log.info("Started the ownership monitor run for activeBrokerCount:{}", brokers.size());
+        }
+
         long startTime = System.nanoTime();
         Set inactiveBrokers = new HashSet<>();
         Set activeBrokers = new HashSet<>(brokers);
@@ -1199,27 +1231,32 @@ protected void monitorOwnerships(List brokers) {
             log.error("Failed to flush the in-flight messages.", e);
         }
 
+        boolean cleaned = false;
         if (serviceUnitTombstoneCleanupCnt > 0) {
             this.totalServiceUnitTombstoneCleanupCnt += serviceUnitTombstoneCleanupCnt;
+            cleaned = true;
         }
 
         if (orphanServiceUnitCleanupCnt > 0) {
             this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt;
+            cleaned = true;
         }
 
-        double monitorTime = TimeUnit.NANOSECONDS
-                .toMillis((System.nanoTime() - startTime));
-        log.info("Completed the ownership monitor run in {} ms. "
-                        + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. "
-                        + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. "
-                        + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. "
-                        + "Approximate cleanupErrorCnt:{}, metrics:{}. ",
-                monitorTime,
-                inactiveBrokers, inactiveBrokers.size(),
-                orphanServiceUnitCleanupCnt,
-                serviceUnitTombstoneCleanupCnt,
-                totalCleanupErrorCntStart - totalCleanupErrorCnt.get(),
-                printCleanupMetrics());
+        if (debug || cleaned) {
+            double monitorTime = TimeUnit.NANOSECONDS
+                    .toMillis((System.nanoTime() - startTime));
+            log.info("Completed the ownership monitor run in {} ms. "
+                            + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. "
+                            + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. "
+                            + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. "
+                            + "Approximate cleanupErrorCnt:{}, metrics:{}. ",
+                    monitorTime,
+                    inactiveBrokers, inactiveBrokers.size(),
+                    orphanServiceUnitCleanupCnt,
+                    serviceUnitTombstoneCleanupCnt,
+                    totalCleanupErrorCntStart - totalCleanupErrorCnt.get(),
+                    printCleanupMetrics());
+        }
 
     }
 
@@ -1353,4 +1390,8 @@ public void listen(StateChangeListener listener) {
     public Set> getOwnershipEntrySet() {
         return tableview.entrySet();
     }
+
+    public static ServiceUnitStateChannel get(PulsarService pulsar) {
+        return ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()).getServiceUnitStateChannel();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java
index 8b373bc5954b2..48665d39a0d3e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java
@@ -75,6 +75,7 @@ public class BrokerLoadData {
      * The historical resource percentage is configured by loadBalancerHistoryResourcePercentage.
      */
     private double weightedMaxEMA;
+    private double msgThroughputEMA;
     private long updatedAt;
 
     @Setter
@@ -88,6 +89,7 @@ public BrokerLoadData() {
         bandwidthOut = new ResourceUsage();
         maxResourceUsage = DEFAULT_RESOURCE_USAGE;
         weightedMaxEMA = DEFAULT_RESOURCE_USAGE;
+        msgThroughputEMA = 0;
     }
 
     /**
@@ -142,6 +144,7 @@ public void update(final BrokerLoadData other) {
         bundleCount = other.bundleCount;
         topics = other.topics;
         weightedMaxEMA = other.weightedMaxEMA;
+        msgThroughputEMA = other.msgThroughputEMA;
         maxResourceUsage = other.maxResourceUsage;
         updatedAt = other.updatedAt;
         reportedAt = other.reportedAt;
@@ -161,6 +164,7 @@ private void updateSystemResourceUsage(final ResourceUsage cpu, final ResourceUs
     private void updateFeatures(ServiceConfiguration conf) {
         updateMaxResourceUsage();
         updateWeightedMaxEMA(conf);
+        updateMsgThroughputEMA(conf);
     }
 
     private void updateMaxResourceUsage() {
@@ -188,6 +192,32 @@ private void updateWeightedMaxEMA(ServiceConfiguration conf) {
                 weightedMaxEMA * historyPercentage + (1 - historyPercentage) * weightedMax;
     }
 
+    private void updateMsgThroughputEMA(ServiceConfiguration conf) {
+        var historyPercentage = conf.getLoadBalancerHistoryResourcePercentage();
+        double msgThroughput = msgThroughputIn + msgThroughputOut;
+        msgThroughputEMA = updatedAt == 0 ? msgThroughput :
+                msgThroughputEMA * historyPercentage + (1 - historyPercentage) * msgThroughput;
+    }
+
+    public void clear() {
+        cpu = new ResourceUsage();
+        memory = new ResourceUsage();
+        directMemory = new ResourceUsage();
+        bandwidthIn = new ResourceUsage();
+        bandwidthOut = new ResourceUsage();
+        maxResourceUsage = DEFAULT_RESOURCE_USAGE;
+        weightedMaxEMA = DEFAULT_RESOURCE_USAGE;
+        msgThroughputEMA = 0;
+        msgThroughputIn = 0;
+        msgThroughputOut = 0;
+        msgRateIn = 0.0;
+        msgRateOut = 0.0;
+        bundleCount = 0;
+        topics = 0;
+        updatedAt = 0;
+        reportedAt = 0;
+    }
+
     public String toString(ServiceConfiguration conf) {
         return String.format("cpu= %.2f%%, memory= %.2f%%, directMemory= %.2f%%, "
                         + "bandwithIn= %.2f%%, bandwithOut= %.2f%%, "
@@ -195,7 +225,7 @@ public String toString(ServiceConfiguration conf) {
                         + "bandwithInResourceWeight= %f, bandwithOutResourceWeight= %f, "
                         + "msgThroughputIn= %.2f, msgThroughputOut= %.2f, msgRateIn= %.2f, msgRateOut= %.2f, "
                         + "bundleCount= %d, "
-                        + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, "
+                        + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, msgThroughputEMA= %.2f, "
                         + "updatedAt= %d, reportedAt= %d",
 
                 cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(),
@@ -207,7 +237,7 @@ public String toString(ServiceConfiguration conf) {
                 conf.getLoadBalancerBandwithOutResourceWeight(),
                 msgThroughputIn, msgThroughputOut, msgRateIn, msgRateOut,
                 bundleCount,
-                maxResourceUsage * 100, weightedMaxEMA * 100,
+                maxResourceUsage * 100, weightedMaxEMA * 100, msgThroughputEMA,
                 updatedAt, reportedAt
         );
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
index d49361741bec4..2f5c32197c1fd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
@@ -86,7 +86,7 @@ public void update(Map bundleStats, int topk) {
             topk = Math.min(topk, arr.size());
             partitionSort(arr, topk);
 
-            for (int i = 0; i < topk; i++) {
+            for (int i = topk - 1; i >= 0; i--) {
                 var etr = arr.get(i);
                 topKBundlesLoadData.add(
                         new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue()));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
index 37483b58b53e1..4a5d41f7576ba 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
@@ -22,8 +22,8 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -68,7 +68,7 @@ Overloaded, new AtomicLong(),
                         Underloaded, new AtomicLong(),
                         Admin, new AtomicLong()),
                 Skip, Map.of(
-                        Balanced, new AtomicLong(),
+                        HitCount, new AtomicLong(),
                         NoBundles, new AtomicLong(),
                         CoolDown, new AtomicLong(),
                         OutDatedData, new AtomicLong(),
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
index e1087ab6e53ba..1301b0708a5bd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
@@ -41,7 +41,7 @@ public enum Label {
     public enum Reason {
         Overloaded,
         Underloaded,
-        Balanced,
+        HitCount,
         NoBundles,
         CoolDown,
         OutDatedData,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java
index 256e52c4554d1..b07acfda7f77d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java
@@ -18,15 +18,21 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.SystemUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerHostUsage;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.GenericBrokerHostUsageImpl;
 import org.apache.pulsar.broker.loadbalance.impl.LinuxBrokerHostUsageImpl;
@@ -37,7 +43,9 @@
  * The broker load data reporter.
  */
 @Slf4j
-public class BrokerLoadDataReporter implements LoadDataReporter {
+public class BrokerLoadDataReporter implements LoadDataReporter, StateChangeListener {
+
+    private static final long TOMBSTONE_DELAY_IN_MILLIS = 1000 * 10;
 
     private final PulsarService pulsar;
 
@@ -54,6 +62,10 @@ public class BrokerLoadDataReporter implements LoadDataReporter
 
     private final BrokerLoadData lastData;
 
+    private volatile long lastTombstonedAt;
+
+    private long tombstoneDelayInMillis;
+
     public BrokerLoadDataReporter(PulsarService pulsar,
                                   String lookupServiceAddress,
                                   LoadDataStore brokerLoadDataStore) {
@@ -68,6 +80,7 @@ public BrokerLoadDataReporter(PulsarService pulsar,
         }
         this.localData = new BrokerLoadData();
         this.lastData = new BrokerLoadData();
+        this.tombstoneDelayInMillis = TOMBSTONE_DELAY_IN_MILLIS;
 
     }
 
@@ -92,8 +105,11 @@ public BrokerLoadData generateLoadData() {
     @Override
     public CompletableFuture reportAsync(boolean force) {
         BrokerLoadData newLoadData = this.generateLoadData();
+        boolean debug = ExtensibleLoadManagerImpl.debug(conf, log);
         if (force || needBrokerDataUpdate()) {
-            log.info("publishing load report:{}", localData.toString(conf));
+            if (debug) {
+                log.info("publishing load report:{}", localData.toString(conf));
+            }
             CompletableFuture future =
                     this.brokerLoadDataStore.pushAsync(this.lookupServiceAddress, newLoadData);
             future.whenComplete((__, ex) -> {
@@ -106,7 +122,9 @@ public CompletableFuture reportAsync(boolean force) {
             });
             return future;
         } else {
-            log.info("skipping load report:{}", localData.toString(conf));
+            if (debug) {
+                log.info("skipping load report:{}", localData.toString(conf));
+            }
         }
         return CompletableFuture.completedFuture(null);
     }
@@ -117,10 +135,13 @@ private boolean needBrokerDataUpdate() {
         final long updateMaxIntervalMillis = TimeUnit.MINUTES
                 .toMillis(loadBalancerReportUpdateMaxIntervalMinutes);
         long timeSinceLastReportWrittenToStore = System.currentTimeMillis() - localData.getReportedAt();
+        boolean debug = ExtensibleLoadManagerImpl.debug(conf, log);
         if (timeSinceLastReportWrittenToStore > updateMaxIntervalMillis) {
-            log.info("Writing local data to metadata store because time since last"
-                            + " update exceeded threshold of {} minutes",
-                    loadBalancerReportUpdateMaxIntervalMinutes);
+            if (debug) {
+                log.info("Writing local data to metadata store because time since last"
+                                + " update exceeded threshold of {} minutes",
+                        loadBalancerReportUpdateMaxIntervalMinutes);
+            }
             // Always update after surpassing the maximum interval.
             return true;
         }
@@ -133,10 +154,13 @@ private boolean needBrokerDataUpdate() {
                                         localData.getMsgThroughputIn() + localData.getMsgThroughputOut()),
                                 percentChange(lastData.getBundleCount(), localData.getBundleCount()))));
         if (maxChange > loadBalancerReportUpdateThresholdPercentage) {
-            log.info("Writing local data to metadata store because maximum change {}% exceeded threshold {}%; "
-                            + "time since last report written is {} seconds", maxChange,
-                    loadBalancerReportUpdateThresholdPercentage,
-                    timeSinceLastReportWrittenToStore / 1000.0);
+            if (debug) {
+                log.info(String.format("Writing local data to metadata store "
+                                + "because maximum change %.2f%% exceeded threshold %d%%. "
+                                + "Time since last report written is %.2f%% seconds", maxChange,
+                        loadBalancerReportUpdateThresholdPercentage,
+                        timeSinceLastReportWrittenToStore / 1000.0));
+            }
             return true;
         }
         return false;
@@ -152,4 +176,50 @@ protected double percentChange(final double oldValue, final double newValue) {
         }
         return 100 * Math.abs((oldValue - newValue) / oldValue);
     }
+
+    @VisibleForTesting
+    protected void tombstone() {
+        var now = System.currentTimeMillis();
+        if (now - lastTombstonedAt < tombstoneDelayInMillis) {
+            return;
+        }
+        var lastSuccessfulTombstonedAt = lastTombstonedAt;
+        lastTombstonedAt = now; // dedup first
+        brokerLoadDataStore.removeAsync(lookupServiceAddress)
+                .whenComplete((__, e) -> {
+                            if (e != null) {
+                                log.error("Failed to clean broker load data.", e);
+                                lastTombstonedAt = lastSuccessfulTombstonedAt;
+                            } else {
+                                boolean debug = ExtensibleLoadManagerImpl.debug(conf, log);
+                                if (debug) {
+                                    log.info("Cleaned broker load data.");
+                                }
+                            }
+                        }
+                );
+
+    }
+
+    @Override
+    public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) {
+        if (t != null) {
+            return;
+        }
+        ServiceUnitState state = ServiceUnitStateData.state(data);
+        switch (state) {
+            case Releasing, Splitting -> {
+                if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) {
+                    localData.clear();
+                    tombstone();
+                }
+            }
+            case Owned -> {
+                if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) {
+                    localData.clear();
+                    tombstone();
+                }
+            }
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
index cc1a39add5d4e..0fa37d3687c20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
@@ -18,10 +18,16 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.concurrent.CompletableFuture;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 
@@ -29,7 +35,9 @@
  * The top k highest-loaded bundles' load data reporter.
  */
 @Slf4j
-public class TopBundleLoadDataReporter implements LoadDataReporter {
+public class TopBundleLoadDataReporter implements LoadDataReporter, StateChangeListener {
+
+    private static final long TOMBSTONE_DELAY_IN_MILLIS = 1000 * 10;
 
     private final PulsarService pulsar;
 
@@ -41,6 +49,9 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore) {
@@ -49,6 +60,7 @@ public TopBundleLoadDataReporter(PulsarService pulsar,
         this.bundleLoadDataStore = bundleLoadDataStore;
         this.lastBundleStatsUpdatedAt = 0;
         this.topKBundles = new TopKBundles(pulsar);
+        this.tombstoneDelayInMillis = TOMBSTONE_DELAY_IN_MILLIS;
     }
 
     @Override
@@ -60,8 +72,7 @@ public TopBundlesLoadData generateLoadData() {
             var pulsarStatsUpdatedAt = pulsarStats.getUpdatedAt();
             if (pulsarStatsUpdatedAt > lastBundleStatsUpdatedAt) {
                 var bundleStats = pulsar.getBrokerService().getBundleStats();
-                double percentage = pulsar.getConfiguration().getLoadBalancerBundleLoadReportPercentage();
-                int topk = Math.max(1, (int) (bundleStats.size() * percentage / 100.0));
+                int topk = pulsar.getConfiguration().getLoadBalancerMaxNumberOfBundlesInBundleLoadReport();
                 topKBundles.update(bundleStats, topk);
                 lastBundleStatsUpdatedAt = pulsarStatsUpdatedAt;
                 result = topKBundles.getLoadData();
@@ -74,6 +85,9 @@ public TopBundlesLoadData generateLoadData() {
     public CompletableFuture reportAsync(boolean force) {
         var topBundlesLoadData = generateLoadData();
         if (topBundlesLoadData != null || force) {
+            if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) {
+                log.info("Reporting TopBundlesLoadData:{}", topKBundles.getLoadData());
+            }
             return this.bundleLoadDataStore.pushAsync(lookupServiceAddress, topKBundles.getLoadData())
                     .exceptionally(e -> {
                         log.error("Failed to report top-bundles load data.", e);
@@ -83,4 +97,47 @@ public CompletableFuture reportAsync(boolean force) {
             return CompletableFuture.completedFuture(null);
         }
     }
+
+    @VisibleForTesting
+    protected void tombstone() {
+        var now = System.currentTimeMillis();
+        if (now - lastTombstonedAt < tombstoneDelayInMillis) {
+            return;
+        }
+        var lastSuccessfulTombstonedAt = lastTombstonedAt;
+        lastTombstonedAt = now; // dedup first
+        bundleLoadDataStore.removeAsync(lookupServiceAddress)
+                .whenComplete((__, e) -> {
+                            if (e != null) {
+                                log.error("Failed to clean broker load data.", e);
+                                lastTombstonedAt = lastSuccessfulTombstonedAt;
+                            } else {
+                                boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log);
+                                if (debug) {
+                                    log.info("Cleaned broker load data.");
+                                }
+                            }
+                        }
+                );
+    }
+
+    @Override
+    public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) {
+        if (t != null) {
+            return;
+        }
+        ServiceUnitState state = ServiceUnitStateData.state(data);
+        switch (state) {
+            case Releasing, Splitting -> {
+                if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) {
+                    tombstone();
+                }
+            }
+            case Owned -> {
+                if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) {
+                    tombstone();
+                }
+            }
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
index 42af396abcad4..421e6240c8f0d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
@@ -20,6 +20,7 @@
 
 import java.util.Map;
 import java.util.Set;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 
@@ -43,4 +44,11 @@ Set findBundlesForUnloading(LoadManagerContext context,
                                                 Map recentlyUnloadedBundles,
                                                 Map recentlyUnloadedBrokers);
 
+    /**
+     * Initializes the internals.
+     *
+     * @param pulsar The pulsar service instance.
+     */
+    void initialize(PulsarService pulsar);
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
index 589df80fc5c14..816fde0038a58 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.StringJoiner;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
@@ -30,6 +31,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
@@ -98,7 +100,7 @@ public SplitScheduler(PulsarService pulsar,
 
     @Override
     public void execute() {
-        boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+        boolean debugMode = ExtensibleLoadManagerImpl.debug(conf, log);
         if (debugMode) {
             log.info("Load balancer enabled: {}, Split enabled: {}.",
                     conf.isLoadBalancerEnabled(), conf.isLoadBalancerAutoBundleSplitEnabled());
@@ -113,8 +115,10 @@ public void execute() {
 
         synchronized (bundleSplitStrategy) {
             final Set decisions = bundleSplitStrategy.findBundlesToSplit(context, pulsar);
+            if (debugMode) {
+                log.info("Split Decisions:", decisions);
+            }
             if (!decisions.isEmpty()) {
-
                 // currently following the unloading timeout
                 var asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs();
                 List> futures = new ArrayList<>();
@@ -156,6 +160,14 @@ public void start() {
         task = loadManagerExecutor.scheduleAtFixedRate(() -> {
             try {
                 execute();
+                var debugMode = ExtensibleLoadManagerImpl.debug(conf, log);
+                if (debugMode) {
+                    StringJoiner joiner = new StringJoiner("\n");
+                    joiner.add("### OwnershipEntrySet start ###");
+                    serviceUnitStateChannel.getOwnershipEntrySet().forEach(e -> joiner.add(e.toString()));
+                    joiner.add("### OwnershipEntrySet end ###");
+                    log.info(joiner.toString());
+                }
             } catch (Throwable e) {
                 log.error("Failed to run the split job.", e);
             }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index effddaf9c4159..669219daba273 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -20,8 +20,8 @@
 
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -30,19 +30,26 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.MinMaxPriorityQueue;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.experimental.Accessors;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
@@ -80,17 +87,23 @@
  * (loadBalancerMaxNumberOfBrokerTransfersPerCycle).
  * 9. Print more logs with a debug option(loadBalancerDebugModeEnabled=true).
  */
+@NoArgsConstructor
 public class TransferShedder implements NamespaceUnloadStrategy {
     private static final Logger log = LoggerFactory.getLogger(TransferShedder.class);
     private static final double KB = 1024;
+    private static final String CANNOT_CONTINUE_UNLOAD_MSG = "Can't continue the unload cycle.";
+    private static final String CANNOT_UNLOAD_BROKER_MSG = "Can't unload broker:%s.";
+    private static final String CANNOT_UNLOAD_BUNDLE_MSG = "Can't unload bundle:%s.";
     private final LoadStats stats = new LoadStats();
-    private final PulsarService pulsar;
-    private final SimpleResourceAllocationPolicies allocationPolicies;
-    private final IsolationPoliciesHelper isolationPoliciesHelper;
-    private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
-
-    private final Set decisionCache;
-    private final UnloadCounter counter;
+    private PulsarService pulsar;
+    private SimpleResourceAllocationPolicies allocationPolicies;
+    private IsolationPoliciesHelper isolationPoliciesHelper;
+    private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+    private Set decisionCache;
+    @Getter
+    private UnloadCounter counter;
+    private ServiceUnitStateChannel channel;
+    private int unloadConditionHitCount = 0;
 
     @VisibleForTesting
     public TransferShedder(UnloadCounter counter){
@@ -110,9 +123,22 @@ public TransferShedder(PulsarService pulsar,
         this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
         this.counter = counter;
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
+        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
         this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
     }
 
+    @Override
+    public void initialize(PulsarService pulsar){
+        this.pulsar = pulsar;
+        this.decisionCache = new HashSet<>();
+        this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
+        var manager = ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get());
+        this.counter = manager.getUnloadCounter();
+        this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
+        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
+        this.antiAffinityGroupPolicyHelper = manager.getAntiAffinityGroupPolicyHelper();
+    }
+
 
     @Getter
     @Accessors(fluent = true)
@@ -122,17 +148,14 @@ static class LoadStats {
         private int totalBrokers;
         private double avg;
         private double std;
-        private MinMaxPriorityQueue minBrokers;
-        private MinMaxPriorityQueue maxBrokers;
         private LoadDataStore loadDataStore;
-
+        private List> brokersSortedByLoad;
+        int maxBrokerIndex;
+        int minBrokerIndex;
+        int numberOfBrokerSheddingPerCycle;
+        int maxNumberOfBrokerSheddingPerCycle;
         LoadStats() {
-            this.minBrokers = MinMaxPriorityQueue.orderedBy((a, b) -> Double.compare(
-                    loadDataStore.get((String) b).get().getWeightedMaxEMA(),
-                    loadDataStore.get((String) a).get().getWeightedMaxEMA())).create();
-            this.maxBrokers = MinMaxPriorityQueue.orderedBy((a, b) -> Double.compare(
-                    loadDataStore.get((String) a).get().getWeightedMaxEMA(),
-                    loadDataStore.get((String) b).get().getWeightedMaxEMA())).create();
+            brokersSortedByLoad = new ArrayList<>();
         }
 
         private void update(double sum, double sqSum, int totalBrokers) {
@@ -143,8 +166,7 @@ private void update(double sum, double sqSum, int totalBrokers) {
             if (totalBrokers == 0) {
                 this.avg = 0;
                 this.std = 0;
-                minBrokers.clear();
-                maxBrokers.clear();
+
             } else {
                 this.avg = sum / totalBrokers;
                 this.std = Math.sqrt(sqSum / totalBrokers - avg * avg);
@@ -156,34 +178,42 @@ void offload(double max, double min, double offload) {
             double maxd = Math.max(0, max - offload);
             double mind = min + offload;
             sqSum += maxd * maxd + mind * mind;
-            std = Math.sqrt(sqSum / totalBrokers - avg * avg);
+            std = Math.sqrt(Math.abs(sqSum / totalBrokers - avg * avg));
+            numberOfBrokerSheddingPerCycle++;
+            minBrokerIndex++;
         }
 
-        void clear(){
+        void clear() {
             sum = 0.0;
             sqSum = 0.0;
             totalBrokers = 0;
             avg = 0.0;
             std = 0.0;
-            minBrokers.clear();
-            maxBrokers.clear();
+            maxBrokerIndex = 0;
+            minBrokerIndex = 0;
+            numberOfBrokerSheddingPerCycle = 0;
+            maxNumberOfBrokerSheddingPerCycle = 0;
+            brokersSortedByLoad.clear();
+            loadDataStore = null;
         }
 
         Optional update(final LoadDataStore loadStore,
+                                               final Map availableBrokers,
                                                Map recentlyUnloadedBrokers,
                                                final ServiceConfiguration conf) {
 
-
+            maxNumberOfBrokerSheddingPerCycle = conf.getLoadBalancerMaxNumberOfBrokerSheddingPerCycle();
+            var debug = ExtensibleLoadManagerImpl.debug(conf, log);
             UnloadDecision.Reason decisionReason = null;
             double sum = 0.0;
             double sqSum = 0.0;
             int totalBrokers = 0;
-            int maxTransfers = conf.getLoadBalancerMaxNumberOfBrokerTransfersPerCycle();
             long now = System.currentTimeMillis();
+            var missingLoadDataBrokers = new HashSet<>(availableBrokers.keySet());
             for (Map.Entry entry : loadStore.entrySet()) {
                 BrokerLoadData localBrokerData = entry.getValue();
                 String broker = entry.getKey();
-
+                missingLoadDataBrokers.remove(broker);
                 // We don't want to use the outdated load data.
                 if (now - localBrokerData.getUpdatedAt()
                         > conf.getLoadBalancerBrokerLoadDataTTLInSeconds() * 1000) {
@@ -196,12 +226,16 @@ Optional update(final LoadDataStore loadS
 
                 // Also, we should give enough time for each broker to recompute its load after transfers.
                 if (recentlyUnloadedBrokers.containsKey(broker)) {
-                    if (localBrokerData.getUpdatedAt() - recentlyUnloadedBrokers.get(broker)
-                            < conf.getLoadBalanceUnloadDelayInSeconds() * 1000) {
-                        log.warn(
-                                "Broker:{} load data timestamp:{} is too early since "
-                                        + "the last transfer timestamp:{}. Stop unloading.",
-                                broker, localBrokerData.getUpdatedAt(), recentlyUnloadedBrokers.get(broker));
+                    var elapsed = localBrokerData.getUpdatedAt() - recentlyUnloadedBrokers.get(broker);
+                    if (elapsed < conf.getLoadBalanceSheddingDelayInSeconds() * 1000) {
+                        if (debug) {
+                            log.warn(
+                                    "Broker:{} load data is too early since "
+                                            + "the last transfer. elapsed {} secs < threshold {} secs",
+                                    broker,
+                                    TimeUnit.MILLISECONDS.toSeconds(elapsed),
+                                    conf.getLoadBalanceSheddingDelayInSeconds());
+                        }
                         update(0.0, 0.0, 0);
                         return Optional.of(CoolDown);
                     } else {
@@ -211,25 +245,28 @@ Optional update(final LoadDataStore loadS
 
                 double load = localBrokerData.getWeightedMaxEMA();
 
-                minBrokers.offer(broker);
-                if (minBrokers.size() > maxTransfers) {
-                    minBrokers.poll();
-                }
-                maxBrokers.offer(broker);
-                if (maxBrokers.size() > maxTransfers) {
-                    maxBrokers.poll();
-                }
                 sum += load;
                 sqSum += load * load;
                 totalBrokers++;
             }
 
-
             if (totalBrokers == 0) {
                 if (decisionReason == null) {
                     decisionReason = NoBrokers;
                 }
                 update(0.0, 0.0, 0);
+                if (debug) {
+                    log.info("There is no broker load data.");
+                }
+                return Optional.of(decisionReason);
+            }
+
+            if (!missingLoadDataBrokers.isEmpty()) {
+                decisionReason = NoLoadData;
+                update(0.0, 0.0, 0);
+                if (debug) {
+                    log.info("There is missing load data from brokers:{}", missingLoadDataBrokers);
+                }
                 return Optional.of(decisionReason);
             }
 
@@ -237,21 +274,35 @@ Optional update(final LoadDataStore loadS
             return Optional.empty();
         }
 
-        boolean hasTransferableBrokers() {
-            return !(maxBrokers.isEmpty() || minBrokers.isEmpty()
-                    || maxBrokers.peekLast().equals(minBrokers().peekLast()));
-        }
-
         void setLoadDataStore(LoadDataStore loadDataStore) {
             this.loadDataStore = loadDataStore;
+            brokersSortedByLoad.addAll(loadDataStore.entrySet());
+            brokersSortedByLoad.sort(Comparator.comparingDouble(
+                    a -> a.getValue().getWeightedMaxEMA()));
+            maxBrokerIndex = brokersSortedByLoad.size() - 1;
+            minBrokerIndex = 0;
+        }
+
+        String peekMinBroker() {
+            return brokersSortedByLoad.get(minBrokerIndex).getKey();
+        }
+
+        String pollMaxBroker() {
+            return brokersSortedByLoad.get(maxBrokerIndex--).getKey();
         }
 
         @Override
         public String toString() {
             return String.format(
-                    "sum:%.2f, sqSum:%.2f, avg:%.2f, std:%.2f, totalBrokers:%d, "
-                            + "minBrokers:%s, maxBrokers:%s",
-                    sum, sqSum, avg, std, totalBrokers, minBrokers, maxBrokers);
+                    "sum:%.2f, sqSum:%.2f, avg:%.2f, std:%.2f, totalBrokers:%d, brokersSortedByLoad:%s",
+                    sum, sqSum, avg, std, totalBrokers,
+                    brokersSortedByLoad.stream().map(v->v.getKey()).collect(Collectors.toList()));
+        }
+
+
+        boolean hasTransferableBrokers() {
+            return numberOfBrokerSheddingPerCycle < maxNumberOfBrokerSheddingPerCycle
+                    && minBrokerIndex < maxBrokerIndex;
         }
     }
 
@@ -263,20 +314,36 @@ public Set findBundlesForUnloading(LoadManagerContext context,
         final var conf = context.brokerConfiguration();
         decisionCache.clear();
         stats.clear();
+        Map availableBrokers;
+        try {
+            availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync()
+                    .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
+        } catch (ExecutionException | InterruptedException | TimeoutException e) {
+            counter.update(Failure, Unknown);
+            log.warn("Failed to fetch available brokers. Stop unloading.", e);
+            return decisionCache;
+        }
 
         try {
             final var loadStore = context.brokerLoadDataStore();
             stats.setLoadDataStore(loadStore);
-            boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+            boolean debugMode = ExtensibleLoadManagerImpl.debug(conf, log);
 
-            var skipReason = stats.update(context.brokerLoadDataStore(), recentlyUnloadedBrokers, conf);
+            var skipReason = stats.update(
+                    context.brokerLoadDataStore(), availableBrokers, recentlyUnloadedBrokers, conf);
             if (skipReason.isPresent()) {
-                log.warn("Failed to update load stat. Reason:{}. Stop unloading.", skipReason.get());
+                if (debugMode) {
+                    log.warn(CANNOT_CONTINUE_UNLOAD_MSG
+                                    + " Skipped the load stat update. Reason:{}.",
+                            skipReason.get());
+                }
                 counter.update(Skip, skipReason.get());
                 return decisionCache;
             }
             counter.updateLoadData(stats.avg, stats.std);
 
+
+
             if (debugMode) {
                 log.info("brokers' load stats:{}", stats);
             }
@@ -287,138 +354,282 @@ public Set findBundlesForUnloading(LoadManagerContext context,
 
             final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd();
             boolean transfer = conf.isLoadBalancerTransferEnabled();
+            if (stats.std() > targetStd || isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
+                unloadConditionHitCount++;
+            } else {
+                unloadConditionHitCount = 0;
+            }
 
-            Map availableBrokers;
-            try {
-                availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync()
-                        .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
-            } catch (ExecutionException | InterruptedException | TimeoutException e) {
-                counter.update(Skip, Unknown);
-                log.warn("Failed to fetch available brokers. Reason: Unknown. Stop unloading.", e);
+            if (unloadConditionHitCount <= conf.getLoadBalancerSheddingConditionHitCountThreshold()) {
+                if (debugMode) {
+                    log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                    + " Shedding condition hit count:{} is less than or equal to the threshold:{}.",
+                            unloadConditionHitCount, conf.getLoadBalancerSheddingConditionHitCountThreshold());
+                }
+                counter.update(Skip, HitCount);
                 return decisionCache;
             }
 
             while (true) {
                 if (!stats.hasTransferableBrokers()) {
                     if (debugMode) {
-                        log.info("Exhausted target transfer brokers. Stop unloading");
+                        log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                + " Exhausted target transfer brokers.");
                     }
                     break;
                 }
                 UnloadDecision.Reason reason;
                 if (stats.std() <= targetStd) {
-                    if (hasMsgThroughput(context, stats.minBrokers.peekLast())) {
+                    if (!isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
                         if (debugMode) {
-                            log.info("std:{} <= targetStd:{} and minBroker:{} has msg throughput. Stop unloading.",
-                                    stats.std, targetStd, stats.minBrokers.peekLast());
+                            log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                            + "The overall cluster load meets the target, std:{} <= targetStd:{},"
+                                            + " and minBroker:{} is not underloaded.",
+                                    stats.std(), targetStd, stats.peekMinBroker());
                         }
                         break;
                     } else {
                         reason = Underloaded;
+                        if (debugMode) {
+                            log.info(String.format("broker:%s is underloaded:%s although "
+                                            + "load std:%.2f <= targetStd:%.2f. "
+                                            + "Continuing unload for this underloaded broker.",
+                                    stats.peekMinBroker(),
+                                    context.brokerLoadDataStore().get(stats.peekMinBroker()).get(),
+                                    stats.std(), targetStd));
+                        }
                     }
                 } else {
                     reason = Overloaded;
                 }
 
-                String maxBroker = stats.maxBrokers().pollLast();
-                String minBroker = stats.minBrokers().pollLast();
+                String maxBroker = stats.pollMaxBroker();
+                String minBroker = stats.peekMinBroker();
                 Optional maxBrokerLoadData = context.brokerLoadDataStore().get(maxBroker);
                 Optional minBrokerLoadData = context.brokerLoadDataStore().get(minBroker);
                 if (maxBrokerLoadData.isEmpty()) {
-                    log.error("maxBroker:{} maxBrokerLoadData is empty. Skip unloading from this max broker.",
-                            maxBroker);
+                    log.error(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " MaxBrokerLoadData is empty.", maxBroker));
                     numOfBrokersWithEmptyLoadData++;
                     continue;
                 }
                 if (minBrokerLoadData.isEmpty()) {
-                    log.error("minBroker:{} minBrokerLoadData is empty. Skip unloading to this min broker.", minBroker);
+                    log.error("Can't transfer load to broker:{}. MinBrokerLoadData is empty.", minBroker);
                     numOfBrokersWithEmptyLoadData++;
                     continue;
                 }
-
-                double max = maxBrokerLoadData.get().getWeightedMaxEMA();
-                double min = minBrokerLoadData.get().getWeightedMaxEMA();
-                double offload = (max - min) / 2;
+                double maxLoad = maxBrokerLoadData.get().getWeightedMaxEMA();
+                double minLoad = minBrokerLoadData.get().getWeightedMaxEMA();
+                double offload = (maxLoad - minLoad) / 2;
                 BrokerLoadData brokerLoadData = maxBrokerLoadData.get();
-                double brokerThroughput = brokerLoadData.getMsgThroughputIn() + brokerLoadData.getMsgThroughputOut();
-                double offloadThroughput = brokerThroughput * offload;
+                double maxBrokerThroughput = brokerLoadData.getMsgThroughputIn()
+                        + brokerLoadData.getMsgThroughputOut();
+                double minBrokerThroughput = minBrokerLoadData.get().getMsgThroughputIn()
+                        + minBrokerLoadData.get().getMsgThroughputOut();
+                double offloadThroughput = maxBrokerThroughput * offload / maxLoad;
 
                 if (debugMode) {
-                    log.info(
-                            "Attempting to shed load from broker:{}{}, which has the max resource "
-                                    + "usage {}%, targetStd:{},"
-                                    + " -- Offloading {}%, at least {} KByte/s of traffic, left throughput {} KByte/s",
+                    log.info(String.format(
+                            "Attempting to shed load from broker:%s%s, which has the max resource "
+                                    + "usage:%.2f%%, targetStd:%.2f,"
+                                    + " -- Trying to offload %.2f%%, %.2f KByte/s of traffic.",
                             maxBroker, transfer ? " to broker:" + minBroker : "",
-                            100 * max, targetStd,
-                            offload * 100, offloadThroughput / KB, (brokerThroughput - offloadThroughput) / KB);
+                            maxLoad * 100,
+                            targetStd,
+                            offload * 100,
+                            offloadThroughput / KB
+                    ));
                 }
 
                 double trafficMarkedToOffload = 0;
-                boolean atLeastOneBundleSelected = false;
+                double trafficMarkedToGain = 0;
 
                 Optional bundlesLoadData = context.topBundleLoadDataStore().get(maxBroker);
                 if (bundlesLoadData.isEmpty() || bundlesLoadData.get().getTopBundlesLoadData().isEmpty()) {
-                    log.error("maxBroker:{} topBundlesLoadData is empty. Skip unloading from this broker.", maxBroker);
+                    log.error(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " TopBundlesLoadData is empty.", maxBroker));
                     numOfBrokersWithEmptyLoadData++;
                     continue;
                 }
 
-                var topBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData();
-                if (topBundlesLoadData.size() > 1) {
-                    int remainingTopBundles = topBundlesLoadData.size();
-                    for (var e : topBundlesLoadData) {
-                        String bundle = e.bundleName();
-                        if (!recentlyUnloadedBundles.containsKey(bundle)
-                                && isTransferable(context, availableBrokers,
-                                bundle, maxBroker, Optional.of(minBroker))) {
-                            var bundleData = e.stats();
-                            double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut;
-                            if (remainingTopBundles > 1
-                                    && (trafficMarkedToOffload < offloadThroughput
-                                    || !atLeastOneBundleSelected)) {
-                                Unload unload;
-                                if (transfer) {
-                                    unload = new Unload(maxBroker, bundle, Optional.of(minBroker));
-                                } else {
-                                    unload = new Unload(maxBroker, bundle);
+                var maxBrokerTopBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData();
+                if (maxBrokerTopBundlesLoadData.size() == 1) {
+                    numOfBrokersWithFewBundles++;
+                    log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG
+                                    + " Sole namespace bundle:%s is overloading the broker. ",
+                            maxBroker, maxBrokerTopBundlesLoadData.iterator().next()));
+                    continue;
+                }
+                Optional minBundlesLoadData = context.topBundleLoadDataStore().get(minBroker);
+                var minBrokerTopBundlesLoadDataIter =
+                        minBundlesLoadData.isPresent() ? minBundlesLoadData.get().getTopBundlesLoadData().iterator() :
+                                null;
+
+
+                if (maxBrokerTopBundlesLoadData.isEmpty()) {
+                    numOfBrokersWithFewBundles++;
+                    log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " Broker overloaded despite having no bundles", maxBroker));
+                    continue;
+                }
+
+                int remainingTopBundles = maxBrokerTopBundlesLoadData.size();
+                for (var e : maxBrokerTopBundlesLoadData) {
+                    String bundle = e.bundleName();
+                    if (channel != null && !channel.isOwner(bundle, maxBroker)) {
+                        if (debugMode) {
+                            log.warn(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                    + " MaxBroker:%s is not the owner.", bundle, maxBroker));
+                        }
+                        continue;
+                    }
+                    if (recentlyUnloadedBundles.containsKey(bundle)) {
+                        if (debugMode) {
+                            log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                            + " Bundle has been recently unloaded at ts:%d.",
+                                    bundle, recentlyUnloadedBundles.get(bundle)));
+                        }
+                        continue;
+                    }
+                    if (!isTransferable(context, availableBrokers, bundle, maxBroker, Optional.of(minBroker))) {
+                        if (debugMode) {
+                            log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                    + " This unload can't meet "
+                                    + "affinity(isolation) or anti-affinity group policies.", bundle));
+                        }
+                        continue;
+                    }
+                    if (remainingTopBundles <= 1) {
+                        if (debugMode) {
+                            log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                            + " The remaining bundles in TopBundlesLoadData from the maxBroker:%s is"
+                                            + " less than or equal to 1.",
+                                    bundle, maxBroker));
+                        }
+                        break;
+                    }
+
+                    var bundleData = e.stats();
+                    double maxBrokerBundleThroughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut;
+                    boolean swap = false;
+                    List minToMaxUnloads = new ArrayList<>();
+                    double minBrokerBundleSwapThroughput = 0.0;
+                    if (trafficMarkedToOffload - trafficMarkedToGain + maxBrokerBundleThroughput > offloadThroughput) {
+                        // see if we can swap bundles from min to max broker to balance better.
+                        if (transfer && minBrokerTopBundlesLoadDataIter != null) {
+                            var maxBrokerNewThroughput =
+                                    maxBrokerThroughput - trafficMarkedToOffload + trafficMarkedToGain
+                                            - maxBrokerBundleThroughput;
+                            var minBrokerNewThroughput =
+                                    minBrokerThroughput + trafficMarkedToOffload - trafficMarkedToGain
+                                            + maxBrokerBundleThroughput;
+                            while (minBrokerTopBundlesLoadDataIter.hasNext()) {
+                                var minBrokerBundleData = minBrokerTopBundlesLoadDataIter.next();
+                                if (!isTransferable(context, availableBrokers,
+                                        minBrokerBundleData.bundleName(), minBroker, Optional.of(maxBroker))) {
+                                    continue;
+                                }
+                                var minBrokerBundleThroughput =
+                                        minBrokerBundleData.stats().msgThroughputIn
+                                                + minBrokerBundleData.stats().msgThroughputOut;
+                                var maxBrokerNewThroughputTmp = maxBrokerNewThroughput + minBrokerBundleThroughput;
+                                var minBrokerNewThroughputTmp = minBrokerNewThroughput - minBrokerBundleThroughput;
+                                if (maxBrokerNewThroughputTmp < maxBrokerThroughput
+                                        && minBrokerNewThroughputTmp < maxBrokerThroughput) {
+                                    minToMaxUnloads.add(new Unload(minBroker,
+                                            minBrokerBundleData.bundleName(), Optional.of(maxBroker)));
+                                    maxBrokerNewThroughput = maxBrokerNewThroughputTmp;
+                                    minBrokerNewThroughput = minBrokerNewThroughputTmp;
+                                    minBrokerBundleSwapThroughput += minBrokerBundleThroughput;
+                                    if (minBrokerNewThroughput <= maxBrokerNewThroughput
+                                            && maxBrokerNewThroughput < maxBrokerThroughput * 0.75) {
+                                        swap = true;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        if (!swap) {
+                            if (debugMode) {
+                                log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                                + " The traffic to unload:%.2f - gain:%.2f = %.2f KByte/s is "
+                                                + "greater than the target :%.2f KByte/s.",
+                                        bundle,
+                                        (trafficMarkedToOffload + maxBrokerBundleThroughput) / KB,
+                                        trafficMarkedToGain / KB,
+                                        (trafficMarkedToOffload - trafficMarkedToGain + maxBrokerBundleThroughput) / KB,
+                                        offloadThroughput / KB));
+                            }
+                            break;
+                        }
+                    }
+                    Unload unload;
+                    if (transfer) {
+                        if (swap) {
+                            minToMaxUnloads.forEach(minToMaxUnload -> {
+                                if (debugMode) {
+                                    log.info("Decided to gain bundle:{} from min broker:{}",
+                                            minToMaxUnload.serviceUnit(), minToMaxUnload.sourceBroker());
                                 }
                                 var decision = new UnloadDecision();
-                                decision.setUnload(unload);
+                                decision.setUnload(minToMaxUnload);
                                 decision.succeed(reason);
                                 decisionCache.add(decision);
-                                trafficMarkedToOffload += throughput;
-                                atLeastOneBundleSelected = true;
-                                remainingTopBundles--;
+                            });
+                            if (debugMode) {
+                                log.info(String.format(
+                                        "Total traffic %.2f KByte/s to transfer from min broker:%s to max broker:%s.",
+                                        minBrokerBundleSwapThroughput / KB, minBroker, maxBroker));
+                                trafficMarkedToGain += minBrokerBundleSwapThroughput;
                             }
                         }
+                        unload = new Unload(maxBroker, bundle, Optional.of(minBroker));
+                    } else {
+                        unload = new Unload(maxBroker, bundle);
                     }
-                    if (!atLeastOneBundleSelected) {
-                        numOfBrokersWithFewBundles++;
+                    var decision = new UnloadDecision();
+                    decision.setUnload(unload);
+                    decision.succeed(reason);
+                    decisionCache.add(decision);
+                    trafficMarkedToOffload += maxBrokerBundleThroughput;
+                    remainingTopBundles--;
+
+                    if (debugMode) {
+                        log.info(String.format("Decided to unload bundle:%s, throughput:%.2f KByte/s."
+                                        + " The traffic marked to unload:%.2f - gain:%.2f = %.2f KByte/s."
+                                        + " Target:%.2f KByte/s.",
+                                bundle, maxBrokerBundleThroughput / KB,
+                                trafficMarkedToOffload / KB,
+                                trafficMarkedToGain / KB,
+                                (trafficMarkedToOffload - trafficMarkedToGain) / KB,
+                                offloadThroughput / KB));
                     }
-                } else if (topBundlesLoadData.size() == 1) {
-                    numOfBrokersWithFewBundles++;
-                    log.warn(
-                            "HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. "
-                                    + "No Load Shedding will be done on this broker",
-                            topBundlesLoadData.iterator().next(), maxBroker);
-                } else {
-                    numOfBrokersWithFewBundles++;
-                    log.warn("Broker {} is overloaded despite having no bundles", maxBroker);
                 }
-
                 if (trafficMarkedToOffload > 0) {
-                    stats.offload(max, min, offload);
+                    var adjustedOffload =
+                            (trafficMarkedToOffload - trafficMarkedToGain) * maxLoad / maxBrokerThroughput;
+                    stats.offload(maxLoad, minLoad, adjustedOffload);
                     if (debugMode) {
                         log.info(
                                 String.format("brokers' load stats:%s, after offload{max:%.2f, min:%.2f, offload:%.2f}",
-                                        stats, max, min, offload));
+                                        stats, maxLoad, minLoad, adjustedOffload));
                     }
+                } else {
+                    numOfBrokersWithFewBundles++;
+                    log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " There is no bundle that can be unloaded in top bundles load data. "
+                            + "Consider splitting bundles owned by the broker "
+                            + "to make each bundle serve less traffic "
+                            + "or increasing loadBalancerMaxNumberOfBundlesInBundleLoadReport"
+                            + " to report more bundles in the top bundles load data.", maxBroker));
                 }
-            }
+
+            } // while end
 
             if (debugMode) {
                 log.info("decisionCache:{}", decisionCache);
             }
+
             if (decisionCache.isEmpty()) {
                 UnloadDecision.Reason reason;
                 if (numOfBrokersWithEmptyLoadData > 0) {
@@ -426,10 +637,13 @@ && isTransferable(context, availableBrokers,
                 } else if (numOfBrokersWithFewBundles > 0) {
                     reason = NoBundles;
                 } else {
-                    reason = Balanced;
+                    reason = HitCount;
                 }
                 counter.update(Skip, reason);
+            } else {
+                unloadConditionHitCount = 0;
             }
+
         } catch (Throwable e) {
             log.error("Failed to process unloading. ", e);
             this.counter.update(Failure, Unknown);
@@ -438,13 +652,19 @@ && isTransferable(context, availableBrokers,
     }
 
 
-    private boolean hasMsgThroughput(LoadManagerContext context, String broker) {
+    private boolean isUnderLoaded(LoadManagerContext context, String broker, double avgLoad) {
         var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker);
         if (brokerLoadDataOptional.isEmpty()) {
             return false;
         }
         var brokerLoadData = brokerLoadDataOptional.get();
-        return brokerLoadData.getMsgThroughputIn() + brokerLoadData.getMsgThroughputOut() > 0.0;
+        if (brokerLoadData.getMsgThroughputEMA() < 1) {
+            return true;
+        }
+
+        return brokerLoadData.getWeightedMaxEMA()
+                < avgLoad * Math.min(0.5, Math.max(0.0,
+                context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2));
     }
 
 
@@ -467,7 +687,8 @@ private boolean isTransferable(LoadManagerContext context,
             return false;
         }
 
-        if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
+        if (antiAffinityGroupPolicyHelper != null
+                && !antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
             return false;
         }
         return true;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
index 31310c5c9cc80..d6c754c90fcf6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
@@ -40,7 +40,6 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
-import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.Reflections;
@@ -81,11 +80,10 @@ public UnloadScheduler(PulsarService pulsar,
                            UnloadManager unloadManager,
                            LoadManagerContext context,
                            ServiceUnitStateChannel channel,
-                           AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper,
                            UnloadCounter counter,
                            AtomicReference> unloadMetrics) {
         this(pulsar, loadManagerExecutor, unloadManager, context, channel,
-                createNamespaceUnloadStrategy(pulsar, antiAffinityGroupPolicyHelper, counter), counter, unloadMetrics);
+                createNamespaceUnloadStrategy(pulsar), counter, unloadMetrics);
     }
 
     @VisibleForTesting
@@ -212,19 +210,22 @@ public void close() {
         this.recentlyUnloadedBrokers.clear();
     }
 
-    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar,
-                                                                         AntiAffinityGroupPolicyHelper helper,
-                                                                         UnloadCounter counter) {
+    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar) {
         ServiceConfiguration conf = pulsar.getConfiguration();
+        NamespaceUnloadStrategy unloadStrategy;
         try {
-            return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class,
+            unloadStrategy = Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(),
+                    NamespaceUnloadStrategy.class,
                     Thread.currentThread().getContextClassLoader());
+            log.info("Created namespace unload strategy:{}", unloadStrategy.getClass().getCanonicalName());
         } catch (Exception e) {
             log.error("Error when trying to create namespace unload strategy: {}",
                     conf.getLoadBalancerLoadPlacementStrategy(), e);
+            log.error("create namespace unload strategy failed. using TransferShedder instead.");
+            unloadStrategy = new TransferShedder();
         }
-        log.error("create namespace unload strategy failed. using TransferShedder instead.");
-        return new TransferShedder(pulsar, counter, helper);
+        unloadStrategy.initialize(pulsar);
+        return unloadStrategy;
     }
 
     private boolean isLoadBalancerSheddingEnabled() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
index da04a287f1ac5..15bfdc747f1fc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -32,6 +32,7 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
@@ -47,15 +48,17 @@
  */
 @Slf4j
 public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleSplitStrategy {
+    private static final String CANNOT_CONTINUE_SPLIT_MSG = "Can't continue the split cycle.";
+    private static final String CANNOT_SPLIT_BUNDLE_MSG = "Can't split broker:%s.";
     private final Set decisionCache;
     private final Map namespaceBundleCount;
-    private final Map bundleHighTrafficFrequency;
+    private final Map splitConditionHitCounts;
     private final SplitCounter counter;
 
     public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) {
         decisionCache = new HashSet<>();
         namespaceBundleCount = new HashMap<>();
-        bundleHighTrafficFrequency = new HashMap<>();
+        splitConditionHitCounts = new HashMap<>();
         this.counter = counter;
 
     }
@@ -71,22 +74,33 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
         long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
         long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
         long maxSplitCount = conf.getLoadBalancerMaxNumberOfBundlesToSplitPerCycle();
-        long splitConditionThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        long splitConditionHitCountThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled();
+        var channel = ServiceUnitStateChannelImpl.get(pulsar);
 
         Map bundleStatsMap = pulsar.getBrokerService().getBundleStats();
         NamespaceBundleFactory namespaceBundleFactory =
                 pulsar.getNamespaceService().getNamespaceBundleFactory();
 
-        // clean bundleHighTrafficFrequency
-        bundleHighTrafficFrequency.keySet().retainAll(bundleStatsMap.keySet());
+        // clean splitConditionHitCounts
+        splitConditionHitCounts.keySet().retainAll(bundleStatsMap.keySet());
 
         for (var entry : bundleStatsMap.entrySet()) {
             final String bundle = entry.getKey();
             final NamespaceBundleStats stats = entry.getValue();
             if (stats.topics < 2) {
                 if (debug) {
-                    log.info("The count of topics on the bundle {} is less than 2, skip split!", bundle);
+                    log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                            + " The topic count is less than 2.", bundle));
+                }
+                continue;
+            }
+
+            if (!channel.isOwner(bundle)) {
+                if (debug) {
+                    log.error(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                            + " This broker is not the owner.", bundle));
+                    counter.update(Failure, Unknown);
                 }
                 continue;
             }
@@ -96,7 +110,8 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
             if (!namespaceBundleFactory
                     .canSplitBundle(namespaceBundleFactory.getBundle(namespaceName, bundleRange))) {
                 if (debug) {
-                    log.info("Can't split the bundle:{}. invalid bundle range:{}. ", bundle, bundleRange);
+                    log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                            + " Invalid bundle range:%s.", bundle, bundleRange));
                 }
                 counter.update(Failure, Unknown);
                 continue;
@@ -117,54 +132,85 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
             }
 
             if (reason != Unknown) {
-                bundleHighTrafficFrequency.put(bundle, bundleHighTrafficFrequency.getOrDefault(bundle, 0) + 1);
+                splitConditionHitCounts.put(bundle, splitConditionHitCounts.getOrDefault(bundle, 0) + 1);
             } else {
-                bundleHighTrafficFrequency.remove(bundle);
+                splitConditionHitCounts.remove(bundle);
+            }
+
+            if (splitConditionHitCounts.getOrDefault(bundle, 0) <= splitConditionHitCountThreshold) {
+                if (debug) {
+                    log.info(String.format(
+                            CANNOT_SPLIT_BUNDLE_MSG
+                                    + " Split condition hit count: %d is"
+                                    + " less than or equal to threshold: %d. "
+                                    + "Topics: %d/%d, "
+                                    + "Sessions: (%d+%d)/%d, "
+                                    + "Message Rate: %.2f/%d (msgs/s), "
+                                    + "Message Throughput: %.2f/%d (MB/s).",
+                            bundle,
+                            splitConditionHitCounts.getOrDefault(bundle, 0),
+                            splitConditionHitCountThreshold,
+                            stats.topics, maxBundleTopics,
+                            stats.producerCount, stats.consumerCount, maxBundleSessions,
+                            totalMessageRate, maxBundleMsgRate,
+                            totalMessageThroughput / LoadManagerShared.MIBI,
+                            maxBundleBandwidth / LoadManagerShared.MIBI
+                    ));
+                }
+                continue;
             }
 
-            if (bundleHighTrafficFrequency.getOrDefault(bundle, 0) > splitConditionThreshold) {
-                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
-                try {
-                    final int bundleCount = pulsar.getNamespaceService()
-                            .getBundleCount(NamespaceName.get(namespace));
-                    if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0))
-                            < maxBundleCount) {
-                        if (debug) {
-                            log.info("The bundle {} is considered to split. Topics: {}/{}, Sessions: ({}+{})/{}, "
-                                            + "Message Rate: {}/{} (msgs/s), Message Throughput: {}/{} (MB/s)",
-                                    bundle, stats.topics, maxBundleTopics, stats.producerCount, stats.consumerCount,
-                                    maxBundleSessions, totalMessageRate, maxBundleMsgRate,
-                                    totalMessageThroughput / LoadManagerShared.MIBI,
-                                    maxBundleBandwidth / LoadManagerShared.MIBI);
-                        }
-                        var decision = new SplitDecision();
-                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
-                        decision.succeed(reason);
-                        decisionCache.add(decision);
-                        int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
-                        namespaceBundleCount.put(namespace, bundleNum + 1);
-                        bundleHighTrafficFrequency.remove(bundle);
-                        // Clear namespace bundle-cache
-                        namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName));
-                        if (decisionCache.size() == maxSplitCount) {
-                            if (debug) {
-                                log.info("Too many bundles to split in this split cycle {} / {}. Stop.",
-                                        decisionCache.size(), maxSplitCount);
-                            }
-                            break;
-                        }
-                    } else {
-                        if (debug) {
-                            log.info(
-                                    "Could not split namespace bundle {} because namespace {} has too many bundles:"
-                                            + "{}", bundle, namespace, bundleCount);
-                        }
+            final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+            try {
+                final int bundleCount = pulsar.getNamespaceService()
+                        .getBundleCount(NamespaceName.get(namespace));
+                if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0))
+                        >= maxBundleCount) {
+                    if (debug) {
+                        log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG + " Namespace:%s has too many bundles:%d",
+                                bundle, namespace, bundleCount));
                     }
-                } catch (Exception e) {
-                    counter.update(Failure, Unknown);
-                    log.warn("Error while computing bundle splits for namespace {}", namespace, e);
+                    continue;
+                }
+            } catch (Exception e) {
+                counter.update(Failure, Unknown);
+                log.warn("Failed to get bundle count in namespace:{}", namespace, e);
+                continue;
+            }
+
+            if (debug) {
+                log.info(String.format(
+                        "Splitting bundle: %s. "
+                                + "Topics: %d/%d, "
+                                + "Sessions: (%d+%d)/%d, "
+                                + "Message Rate: %.2f/%d (msgs/s), "
+                                + "Message Throughput: %.2f/%d (MB/s)",
+                        bundle,
+                        stats.topics, maxBundleTopics,
+                        stats.producerCount, stats.consumerCount, maxBundleSessions,
+                        totalMessageRate, maxBundleMsgRate,
+                        totalMessageThroughput / LoadManagerShared.MIBI,
+                        maxBundleBandwidth / LoadManagerShared.MIBI
+                ));
+            }
+            var decision = new SplitDecision();
+            decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
+            decision.succeed(reason);
+            decisionCache.add(decision);
+            int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
+            namespaceBundleCount.put(namespace, bundleNum + 1);
+            splitConditionHitCounts.remove(bundle);
+            // Clear namespace bundle-cache
+            namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName));
+            if (decisionCache.size() == maxSplitCount) {
+                if (debug) {
+                    log.info(CANNOT_CONTINUE_SPLIT_MSG
+                                    + "Too many bundles split in this cycle {} / {}.",
+                            decisionCache.size(), maxSplitCount);
                 }
+                break;
             }
+
         }
         return decisionCache;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
index 678927dac9293..902cfdaf73f5f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
@@ -82,7 +82,7 @@ public Optional select(
             Set candidates, ServiceUnitId bundleToAssign, LoadManagerContext context) {
         var conf = context.brokerConfiguration();
         if (candidates.isEmpty()) {
-            log.info("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign);
+            log.warn("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign);
             return Optional.empty();
         }
 
@@ -131,8 +131,10 @@ public Optional select(
 
         if (bestBrokers.isEmpty()) {
             // Assign randomly as all brokers are overloaded.
-            log.warn("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, "
-                    + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size());
+            if (debugMode) {
+                log.info("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, "
+                        + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size());
+            }
             for (String broker : candidates) {
                 bestBrokers.add(broker);
             }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index f8a7a9b629f4e..4c57e6b93fa3c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -26,8 +26,8 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -671,7 +671,7 @@ public void testGetMetrics() throws Exception {
                         put(Underloaded, new AtomicLong(2));
                     }},
                     Skip, new LinkedHashMap<>() {{
-                        put(Balanced, new AtomicLong(3));
+                        put(HitCount, new AtomicLong(3));
                         put(NoBundles, new AtomicLong(4));
                         put(CoolDown, new AtomicLong(5));
                         put(OutDatedData, new AtomicLong(6));
@@ -756,7 +756,7 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
                         dimensions=[{broker=localhost, feature=max, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=0.04}]
                         dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_unload_broker_total=2, brk_lb_unload_bundle_total=3}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Failure}], metrics=[{brk_lb_unload_broker_breakdown_total=10}]
-                        dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}]
+                        dimensions=[{broker=localhost, metric=bundleUnloading, reason=HitCount, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=CoolDown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=5}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=OutDatedData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=6}]
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index c05c8ac741565..9eda98e5d842d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -78,6 +78,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
@@ -132,6 +133,8 @@ protected void setup() throws Exception {
         pulsar1 = pulsar;
         registry = new BrokerRegistryImpl(pulsar);
         loadManagerContext = mock(LoadManagerContext.class);
+        doReturn(mock(LoadDataStore.class)).when(loadManagerContext).brokerLoadDataStore();
+        doReturn(mock(LoadDataStore.class)).when(loadManagerContext).topBundleLoadDataStore();
         brokerSelector = mock(BrokerSelectionStrategy.class);
         additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf());
         pulsar2 = additionalPulsarTestContext.getPulsarService();
@@ -1208,6 +1211,69 @@ public void splitTestWhenDestBrokerFails()
 
     }
 
+    @Test(priority = 15)
+    public void testIsOwner() throws IllegalAccessException {
+
+        var owner1 = channel1.isOwner(bundle);
+        var owner2 = channel2.isOwner(bundle);
+
+        assertFalse(owner1);
+        assertFalse(owner2);
+
+        owner1 = channel1.isOwner(bundle, lookupServiceAddress2);
+        owner2 = channel2.isOwner(bundle, lookupServiceAddress1);
+
+        assertFalse(owner1);
+        assertFalse(owner2);
+
+        channel1.publishAssignEventAsync(bundle, lookupServiceAddress1);
+        owner2 = channel2.isOwner(bundle);
+        assertFalse(owner2);
+
+        waitUntilOwnerChanges(channel1, bundle, null);
+        waitUntilOwnerChanges(channel2, bundle, null);
+
+        owner1 = channel1.isOwner(bundle);
+        owner2 = channel2.isOwner(bundle);
+
+        assertTrue(owner1);
+        assertFalse(owner2);
+
+        owner1 = channel1.isOwner(bundle, lookupServiceAddress1);
+        owner2 = channel2.isOwner(bundle, lookupServiceAddress2);
+
+        assertTrue(owner1);
+        assertFalse(owner2);
+
+        owner1 = channel2.isOwner(bundle, lookupServiceAddress1);
+        owner2 = channel1.isOwner(bundle, lookupServiceAddress2);
+
+        assertTrue(owner1);
+        assertFalse(owner2);
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, lookupServiceAddress1, 1));
+        assertTrue(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, lookupServiceAddress1, 1));
+        assertTrue(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, null);
+        assertFalse(channel1.isOwner(bundle));
+    }
+
+
     private static ConcurrentOpenHashMap>> getOwnerRequests(
             ServiceUnitStateChannel channel) throws IllegalAccessException {
         return (ConcurrentOpenHashMap>>)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java
index 5ba7629dd1132..85792a7ba9387 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java
@@ -74,6 +74,7 @@ public void testUpdateBySystemResourceUsage() {
         assertEquals(data.getTopics(), 6);
         assertEquals(data.getMaxResourceUsage(), 0.04); // skips memory usage
         assertEquals(data.getWeightedMaxEMA(), 2);
+        assertEquals(data.getMsgThroughputEMA(), 3);
         assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now));
 
         now = System.currentTimeMillis();
@@ -103,6 +104,7 @@ public void testUpdateBySystemResourceUsage() {
         assertEquals(data.getTopics(), 10);
         assertEquals(data.getMaxResourceUsage(), 3.0);
         assertEquals(data.getWeightedMaxEMA(), 1.875);
+        assertEquals(data.getMsgThroughputEMA(), 5);
         assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now));
         assertEquals(data.getReportedAt(), 0l);
         assertEquals(data.toString(conf), "cpu= 300.00%, memory= 100.00%, directMemory= 2.00%, "
@@ -111,8 +113,11 @@ public void testUpdateBySystemResourceUsage() {
                 + "bandwithInResourceWeight= 0.500000, bandwithOutResourceWeight= 0.500000, "
                 + "msgThroughputIn= 5.00, msgThroughputOut= 6.00, "
                 + "msgRateIn= 7.00, msgRateOut= 8.00, bundleCount= 9, "
-                + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, "
+                + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, msgThroughputEMA= 5.00, "
                 + "updatedAt= " + data.getUpdatedAt() + ", reportedAt= " + data.getReportedAt());
+
+        data.clear();
+        assertEquals(data, new BrokerLoadData());
     }
 
     @Test
@@ -143,6 +148,9 @@ public void testUpdateByBrokerLoadData() {
         data.update(other);
 
         assertEquals(data, other);
+
+        data.clear();
+        assertEquals(data, new BrokerLoadData());
     }
 
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
index 9b42163bd664b..472d44df8906d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
@@ -87,15 +87,15 @@ public void testTopBundlesLoadData() {
         Map bundleStats = new HashMap<>();
         var topKBundles = new TopKBundles(pulsar);
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
-        stats1.msgRateIn = 500;
+        stats1.msgRateIn = 100000;
         bundleStats.put(bundle1, stats1);
 
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
-        stats2.msgRateIn = 10000;
+        stats2.msgRateIn = 500;
         bundleStats.put(bundle2, stats2);
 
         NamespaceBundleStats stats3 = new NamespaceBundleStats();
-        stats3.msgRateIn = 100000;
+        stats3.msgRateIn = 10000;
         bundleStats.put(bundle3, stats3);
 
         NamespaceBundleStats stats4 = new NamespaceBundleStats();
@@ -107,8 +107,8 @@ public void testTopBundlesLoadData() {
         var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
         var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2);
 
-        assertEquals(top0.bundleName(), bundle3);
-        assertEquals(top1.bundleName(), bundle2);
+        assertEquals(top0.bundleName(), bundle2);
+        assertEquals(top1.bundleName(), bundle3);
         assertEquals(top2.bundleName(), bundle1);
     }
 
@@ -225,8 +225,8 @@ public void testLoadBalancerSheddingBundlesWithPoliciesEnabledConfig() throws Me
         var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
         var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
 
-        assertEquals(top0.bundleName(), bundle2);
-        assertEquals(top1.bundleName(), bundle1);
+        assertEquals(top0.bundleName(), bundle1);
+        assertEquals(top1.bundleName(), bundle2);
 
         configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
index ee7e708667a32..93ab35981e1c4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
@@ -18,28 +18,37 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.broker.stats.BrokerStats;
+import org.apache.pulsar.client.util.ExecutorProvider;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -52,17 +61,24 @@ public class BrokerLoadDataReporterTest {
     ServiceConfiguration config;
     BrokerStats brokerStats;
     SystemResourceUsage usage;
+    String broker = "broker1";
+    String bundle = "bundle1";
+    ScheduledExecutorService executor;
 
     @BeforeMethod
     void setup() {
         config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
         pulsar = mock(PulsarService.class);
         store = mock(LoadDataStore.class);
         brokerService = mock(BrokerService.class);
         pulsarStats = mock(PulsarStats.class);
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
-        doReturn(Executors.newSingleThreadScheduledExecutor()).when(pulsar).getLoadManagerExecutor();
+        executor = Executors
+                .newSingleThreadScheduledExecutor(new
+                        ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager"));
+        doReturn(executor).when(pulsar).getLoadManagerExecutor();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         brokerStats = new BrokerStats(0);
         brokerStats.topics = 6;
@@ -74,6 +90,7 @@ void setup() {
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(brokerStats).when(pulsarStats).getBrokerStats();
         doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any());
+        doReturn(CompletableFuture.completedFuture(null)).when(store).removeAsync(any());
 
         usage = new SystemResourceUsage();
         usage.setCpu(new ResourceUsage(1.0, 100.0));
@@ -83,6 +100,11 @@ void setup() {
         usage.setBandwidthOut(new ResourceUsage(4.0, 100.0));
     }
 
+    @AfterMethod
+    void shutdown(){
+        executor.shutdown();
+    }
+
     public void testGenerate() throws IllegalAccessException {
         try (MockedStatic mockLoadManagerShared = Mockito.mockStatic(LoadManagerShared.class)) {
             mockLoadManagerShared.when(() -> LoadManagerShared.getSystemResourceUsage(any())).thenReturn(usage);
@@ -124,4 +146,73 @@ public void testReport() throws IllegalAccessException {
         }
     }
 
+    @Test
+    public void testTombstone() throws IllegalAccessException, InterruptedException {
+
+        var target = spy(new BrokerLoadDataReporter(pulsar, broker, store));
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Assigning, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Deleted, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Init, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Free, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT),
+                new RuntimeException());
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(1)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(2)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+
+        FieldUtils.writeDeclaredField(target, "tombstoneDelayInMillis", 0, true);
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(3)).tombstone();
+            verify(store, times(2)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(4)).tombstone();
+            verify(store, times(3)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
index b5c415c405fbd..be8c6af2b0404 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
@@ -18,9 +18,12 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
@@ -29,9 +32,12 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
@@ -42,6 +48,7 @@
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -58,10 +65,13 @@ public class TopBundleLoadDataReporterTest {
     private LocalPoliciesResources localPoliciesResources;
     String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF";
     String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF";
+    String bundle = bundle1;
+    String broker = "broker-1";
 
     @BeforeMethod
     void setup() throws MetadataStoreException {
         config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
         pulsar = mock(PulsarService.class);
         store = mock(LoadDataStore.class);
         brokerService = mock(BrokerService.class);
@@ -75,6 +85,7 @@ void setup() throws MetadataStoreException {
         doReturn(config).when(pulsar).getConfiguration();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any());
+        doReturn(CompletableFuture.completedFuture(null)).when(store).removeAsync(any());
 
         doReturn(pulsarResources).when(pulsar).getPulsarResources();
         doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
@@ -102,22 +113,23 @@ public void testZeroUpdatedAt() {
 
     public void testGenerateLoadData() throws IllegalAccessException {
         doReturn(1l).when(pulsarStats).getUpdatedAt();
-        config.setLoadBalancerBundleLoadReportPercentage(100);
+        config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(2);
         var target = new TopBundleLoadDataReporter(pulsar, "", store);
         var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 2);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
-        config.setLoadBalancerBundleLoadReportPercentage(50);
+        config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(1);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
         expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
-        config.setLoadBalancerBundleLoadReportPercentage(1);
+        config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(0);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
+
         expected = new TopKBundles(pulsar);
-        expected.update(bundleStats, 1);
+        expected.update(bundleStats, 0);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         doReturn(new HashMap()).when(brokerService).getBundleStats();
@@ -128,21 +140,83 @@ public void testGenerateLoadData() throws IllegalAccessException {
 
 
     public void testReportForce()  {
-        var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store);
+        var target = new TopBundleLoadDataReporter(pulsar, broker, store);
         target.reportAsync(false);
         verify(store, times(0)).pushAsync(any(), any());
         target.reportAsync(true);
-        verify(store, times(1)).pushAsync("broker-1", new TopBundlesLoadData());
+        verify(store, times(1)).pushAsync(broker, new TopBundlesLoadData());
 
     }
 
     public void testReport(){
-        var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store);
+        pulsar.getConfiguration().setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(1);
+        var target = new TopBundleLoadDataReporter(pulsar, broker, store);
         doReturn(1l).when(pulsarStats).getUpdatedAt();
         var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         target.reportAsync(false);
-        verify(store, times(1)).pushAsync("broker-1", expected.getLoadData());
+        verify(store, times(1)).pushAsync(broker, expected.getLoadData());
     }
 
+    @Test
+    public void testTombstone() throws IllegalAccessException {
+
+        var target = spy(new TopBundleLoadDataReporter(pulsar, broker, store));
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Assigning, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Deleted, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Init, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Free, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT),
+                new RuntimeException());
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(1)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(2)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+        });
+
+        FieldUtils.writeDeclaredField(target, "tombstoneDelayInMillis", 0, true);
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(3)).tombstone();
+            verify(store, times(2)).removeAsync(eq(broker));
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(4)).tombstone();
+            verify(store, times(3)).removeAsync(eq(broker));
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 5ca9345d6def5..8c8d18e202a00 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -18,10 +18,11 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -52,6 +53,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
@@ -59,7 +61,11 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
@@ -97,6 +103,10 @@ public class TransferShedderTest {
     double setupLoadStd = 0.3982762860126121;
 
     PulsarService pulsar;
+    NamespaceService namespaceService;
+    ExtensibleLoadManagerWrapper loadManagerWrapper;
+    ExtensibleLoadManagerImpl loadManager;
+    ServiceUnitStateChannel channel;
     ServiceConfiguration conf;
     AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     LocalPoliciesResources localPoliciesResources;
@@ -108,13 +118,16 @@ public class TransferShedderTest {
     @BeforeMethod
     public void init() throws MetadataStoreException {
         pulsar = mock(PulsarService.class);
+        loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class);
+        loadManager = mock(ExtensibleLoadManagerImpl.class);
+        channel = mock(ServiceUnitStateChannelImpl.class);
         conf = new ServiceConfiguration();
         conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
         var pulsarResources = mock(PulsarResources.class);
         var namespaceResources = mock(NamespaceResources.class);
         var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
         var factory = mock(NamespaceBundleFactory.class);
-        var namespaceService = mock(NamespaceService.class);
+        namespaceService = mock(NamespaceService.class);
         localPoliciesResources = mock(LocalPoliciesResources.class);
         antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
         doReturn(conf).when(pulsar).getConfiguration();
@@ -136,25 +149,30 @@ public void init() throws MetadataStoreException {
                     (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
             return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory);
         }).when(factory).getBundle(anyString(), anyString());
-        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
+        doReturn(new AtomicReference<>(loadManagerWrapper)).when(pulsar).getLoadManager();
+        doReturn(loadManager).when(loadManagerWrapper).get();
+        doReturn(channel).when(loadManager).getServiceUnitStateChannel();
+        doReturn(true).when(channel).isOwner(any(), any());
     }
 
     public LoadManagerContext setupContext(){
         var ctx = getContext();
 
-        var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  2));
-        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx,  4));
-        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  6));
-        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  80));
-        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  90));
+
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 3000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 4000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 6000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 7000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000));
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000));
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2"));
+        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3"));
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5"));
         return ctx;
     }
 
@@ -166,8 +184,8 @@ public LoadManagerContext setupContext(int clusterSize) {
 
         Random rand = new Random();
         for (int i = 0; i < clusterSize; i++) {
-            int brokerLoad = rand.nextInt(100);
-            brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad));
+            int brokerLoad = rand.nextInt(1000);
+            brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
             int bundleLoad = rand.nextInt(brokerLoad + 1);
             topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
                     bundleLoad, brokerLoad - bundleLoad));
@@ -175,7 +193,7 @@ public LoadManagerContext setupContext(int clusterSize) {
         return ctx;
     }
 
-    public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) {
+    public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load, String broker) {
         var loadData = new BrokerLoadData();
         SystemResourceUsage usage1 = new SystemResourceUsage();
         var cpu = new ResourceUsage(load, 100.0);
@@ -188,8 +206,17 @@ public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) {
         usage1.setDirectMemory(directMemory);
         usage1.setBandwidthIn(bandwidthIn);
         usage1.setBandwidthOut(bandwidthOut);
-        loadData.update(usage1, 1,2,3,4,5,6,
-                ctx.brokerConfiguration());
+        if (ctx.topBundleLoadDataStore()
+                .get(broker).isPresent()) {
+            var throughputOut = ctx.topBundleLoadDataStore()
+                    .get(broker).get()
+                    .getTopBundlesLoadData().stream().mapToDouble(v -> v.stats().msgThroughputOut).sum();
+            loadData.update(usage1, 1, throughputOut, 3, 4, 5, 6,
+                    ctx.brokerConfiguration());
+        } else {
+            loadData.update(usage1, 1, 2, 3, 4, 5, 6,
+                    ctx.brokerConfiguration());
+        }
         return loadData;
     }
 
@@ -204,6 +231,29 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
         return topKBundles.getLoadData();
     }
 
+    public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix,
+                                                int load1, int load2, int load3, int load4, int load5) {
+        var namespaceBundleStats1 = new NamespaceBundleStats();
+        namespaceBundleStats1.msgThroughputOut = load1;
+        var namespaceBundleStats2 = new NamespaceBundleStats();
+        namespaceBundleStats2.msgThroughputOut = load2;
+        var namespaceBundleStats3 = new NamespaceBundleStats();
+        namespaceBundleStats3.msgThroughputOut = load3;
+        var namespaceBundleStats4 = new NamespaceBundleStats();
+        namespaceBundleStats4.msgThroughputOut = load4;
+        var namespaceBundleStats5 = new NamespaceBundleStats();
+        namespaceBundleStats5.msgThroughputOut = load5;
+        var topKBundles = new TopKBundles(pulsar);
+        topKBundles.update(Map.of(
+                bundlePrefix + "/0x00000000_0x1FFFFFFF", namespaceBundleStats1,
+                bundlePrefix + "/0x1FFFFFFF_0x2FFFFFFF", namespaceBundleStats2,
+                bundlePrefix + "/0x2FFFFFFF_0x3FFFFFFF", namespaceBundleStats3,
+                bundlePrefix + "/0x3FFFFFFF_0x4FFFFFFF", namespaceBundleStats4,
+                bundlePrefix + "/0x4FFFFFFF_0x5FFFFFFF", namespaceBundleStats5
+        ), 5);
+        return topKBundles.getLoadData();
+    }
+
     public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) {
         var namespaceBundleStats1 = new NamespaceBundleStats();
         namespaceBundleStats1.msgThroughputOut = load1;
@@ -230,6 +280,7 @@ public LoadManagerContext getContext(){
         var conf = new ServiceConfiguration();
         conf.setLoadBalancerDebugModeEnabled(true);
         conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+        conf.setLoadBalancerSheddingConditionHitCountThreshold(0);
         var brokerLoadDataStore = new LoadDataStore() {
             Map map = new HashMap<>();
             @Override
@@ -357,29 +408,44 @@ public void testEmptyBrokerLoadData() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         assertTrue(res.isEmpty());
         assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBrokers).get(), 1);
     }
 
+    @Test
+    public void testNoOwnerLoadData() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        FieldUtils.writeDeclaredField(transferShedder, "channel", channel, true);
+        var ctx = setupContext();
+        doReturn(false).when(channel).isOwner(any(), any());
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
     @Test
     public void testEmptyTopBundlesLoadData() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
 
-        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  90));
-        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx,  10));
-        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  20));
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2"));
+        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3"));
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5"));
+
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         assertTrue(res.isEmpty());
         assertEquals(counter.getBreakdownCounters().get(Skip).get(NoLoadData).get(), 1);
-        assertEquals(counter.getLoadAvg(), 0.39999999999999997);
-        assertEquals(counter.getLoadStd(), 0.35590260840104376);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
@@ -445,16 +511,18 @@ public void testRecentlyUnloadedBundles() {
         recentlyUnloadedBundles.put(bundleD1, now);
         recentlyUnloadedBundles.put(bundleD2, now);
         var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of());
-
-        assertTrue(res.isEmpty());
-        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker3",
+                "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF",
+                Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(res, expected);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testGetAvailableBrokersFailed() {
-        var pulsar = getMockPulsar();
         UnloadCounter counter = new UnloadCounter();
         AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
         TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper);
@@ -462,17 +530,16 @@ public void testGetAvailableBrokersFailed() {
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1);
-        assertEquals(counter.getLoadAvg(), setupLoadAvg);
-        assertEquals(counter.getLoadStd(), setupLoadStd);
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
+        assertEquals(counter.getLoadAvg(), 0.0);
+        assertEquals(counter.getLoadStd(), 0.0);
     }
 
     @Test(timeOut = 30 * 1000)
     public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
-        var pulsar = getMockPulsar();
+        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper));
-
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
         FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true);
@@ -484,7 +551,7 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
         var ctx = setupContext();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected = new HashSet();
-        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")),
                 Success, Overloaded));
         assertEquals(res, expected);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
@@ -572,29 +639,9 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies,
         }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt());
     }
 
-    private PulsarService getMockPulsar() {
-        var pulsar = mock(PulsarService.class);
-        var namespaceService = mock(NamespaceService.class);
-        doReturn(namespaceService).when(pulsar).getNamespaceService();
-        NamespaceBundleFactory factory = mock(NamespaceBundleFactory.class);
-        doReturn(factory).when(namespaceService).getNamespaceBundleFactory();
-        doAnswer(answer -> {
-            String namespace = answer.getArgument(0, String.class);
-            String bundleRange = answer.getArgument(1, String.class);
-            String[] boundaries = bundleRange.split("_");
-            Long lowerEndpoint = Long.decode(boundaries[0]);
-            Long upperEndpoint = Long.decode(boundaries[1]);
-            Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint,
-                    (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
-            return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory);
-        }).when(factory).getBundle(anyString(), anyString());
-        return pulsar;
-    }
-
 
     @Test
     public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
-        var pulsar = getMockPulsar();
         var counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper);
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
@@ -606,7 +653,10 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any());
 
         var ctx = setupContext();
-        doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
+        var antiAffinityGroupPolicyHelperSpy = (AntiAffinityGroupPolicyHelper)
+                spy(FieldUtils.readDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", true));
+        doReturn(false).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), any(), any(), any());
+        FieldUtils.writeDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", antiAffinityGroupPolicyHelperSpy, true);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         assertTrue(res.isEmpty());
@@ -615,7 +665,7 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         assertEquals(counter.getLoadStd(), setupLoadStd);
 
 
-        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any());
+        doReturn(true).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), eq(bundleE1), any(), any());
         var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected2 = new HashSet<>();
         expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
@@ -630,11 +680,18 @@ public void testTargetStd() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
+        BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class),
+                "broker3", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  10));
-        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx,  20));
-        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  30));
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 20, "broker2"));
+        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 30, "broker3"));
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
 
@@ -645,7 +702,7 @@ public void testTargetStd() {
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         assertTrue(res.isEmpty());
-        assertEquals(counter.getBreakdownCounters().get(Skip).get(Balanced).get(), 1);
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(HitCount).get(), 1);
         assertEquals(counter.getLoadAvg(), 0.2000000063578288);
         assertEquals(counter.getLoadStd(), 0.08164966587949089);
     }
@@ -669,6 +726,26 @@ public void testSingleTopBundlesLoadData() {
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
+    @Test
+    public void testBundleThroughputLargerThanOffloadThreshold() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000));
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker3",
+                "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF",
+                Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
 
     @Test
     public void testTargetStdAfterTransfer() {
@@ -676,8 +753,8 @@ public void testTargetStdAfterTransfer() {
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
-        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65));
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65, "broker5"));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new HashSet();
@@ -688,19 +765,186 @@ public void testTargetStdAfterTransfer() {
         assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
+    @Test
+    public void testUnloadBundlesGreaterThanTargetThroughput() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000));
+        topBundlesLoadDataStore.pushAsync("broker2",
+                getTopBundlesLoad("my-tenant/my-namespaceB", 100000000, 180000000, 220000000, 250000000, 250000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2"));
+
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        assertEquals(counter.getLoadAvg(), 5.05);
+        assertEquals(counter.getLoadStd(), 4.95);
+        assertEquals(res, expected);
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.std(), 0.050000004900021836);
+    }
+
+    @Test
+    public void testSkipBundlesGreaterThanTargetThroughputAfterSplit() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+        ctx.brokerConfiguration().setLoadBalancerBrokerLoadTargetStd(0.20);
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1",
+                getTopBundlesLoad("my-tenant/my-namespaceA", 1, 500000000));
+        topBundlesLoadDataStore.pushAsync("broker2",
+                getTopBundlesLoad("my-tenant/my-namespaceB", 500000000, 500000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 50, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2"));
+
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+    }
+
+
+    @Test
+    public void testUnloadBundlesLessThanTargetThroughputAfterSplit() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2"));
+
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(counter.getLoadAvg(), 5.05);
+        assertEquals(counter.getLoadStd(), 4.95);
+        assertEquals(res, expected);
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.std(), 0.050000004900021836);
+
+    }
+
+
+    @Test
+    public void testUnloadBundlesGreaterThanTargetThroughputAfterSplit() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 48, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2"));
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker1",
+                        res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1")).findFirst().get()
+                        .getUnload().serviceUnit(), Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker2",
+                        res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2")).findFirst().get()
+                                .getUnload().serviceUnit(), Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(counter.getLoadAvg(), 0.74);
+        assertEquals(counter.getLoadStd(), 0.26);
+        assertEquals(res, expected);
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.std(), 2.5809568279517847E-8);
+    }
+
+
     @Test
     public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
-        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65));
 
-        var load = getCpuLoad(ctx,  4);
-        FieldUtils.writeDeclaredField(load,"msgThroughputIn", 0, true);
-        FieldUtils.writeDeclaredField(load,"msgThroughputOut", 0, true);
+        var load = getCpuLoad(ctx,  4, "broker2");
+        FieldUtils.writeDeclaredField(load,"msgThroughputEMA", 0, true);
         brokerLoadDataStore.pushAsync("broker2", load);
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65, "broker5"));
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
@@ -714,13 +958,63 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
         assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
+    @Test
+    public void testMinBrokerWithLowerLoadThanAvg() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+
+        var load = getCpuLoad(ctx,  3 , "broker2");
+        brokerLoadDataStore.pushAsync("broker2", load);
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65, "broker5"));
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Underloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.262);
+        assertEquals(counter.getLoadStd(), 0.2780935094532054);
+    }
+
     @Test
     public void testMaxNumberOfTransfersPerShedderCycle() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         ctx.brokerConfiguration()
-                .setLoadBalancerMaxNumberOfBrokerTransfersPerCycle(10);
+                .setLoadBalancerMaxNumberOfBrokerSheddingPerCycle(10);
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
+    @Test
+    public void testLoadBalancerSheddingConditionHitCountThreshold() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+        int max = 3;
+        ctx.brokerConfiguration()
+                .setLoadBalancerSheddingConditionHitCountThreshold(max);
+        for (int i = 0; i < max; i++) {
+            var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+            assertTrue(res.isEmpty());
+            assertEquals(counter.getBreakdownCounters().get(Skip).get(HitCount).get(), i+1);
+            assertEquals(counter.getLoadAvg(), setupLoadAvg);
+            assertEquals(counter.getLoadStd(), setupLoadStd);
+        }
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected = new HashSet();
         expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
@@ -738,7 +1032,7 @@ public void testRemainingTopBundles() {
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new HashSet();
@@ -751,6 +1045,33 @@ public void testRemainingTopBundles() {
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
+    @Test
+    public void testLoadMoreThan100() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  200, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  1000, "broker5"));
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 2.4240000000000004);
+        assertEquals(counter.getLoadStd(), 3.8633332758124816);
+
+
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.avg(), 2.4240000000000004);
+        assertEquals(stats.std(), 2.781643776903451);
+    }
+
     @Test
     public void testRandomLoad() throws IllegalAccessException {
         UnloadCounter counter = new UnloadCounter();
@@ -767,7 +1088,7 @@ public void testRandomLoad() throws IllegalAccessException {
     }
 
     @Test
-    public void testLoadStats() {
+    public void testRandomLoadStats() {
         int numBrokers = 10;
         double delta = 0.0001;
         for (int t = 0; t < 5; t++) {
@@ -776,8 +1097,13 @@ public void testLoadStats() {
             var loadStore = ctx.brokerLoadDataStore();
             stats.setLoadDataStore(loadStore);
             var conf = ctx.brokerConfiguration();
-            stats.update(loadStore, Map.of(), conf);
             double[] loads = new double[numBrokers];
+            final Map availableBrokers = new HashMap<>();
+            for (int i = 0; i < loads.length; i++) {
+                availableBrokers.put("broker" + i, mock(BrokerLookupData.class));
+            }
+            stats.update(loadStore, availableBrokers, Map.of(), conf);
+
             var brokerLoadDataStore = ctx.brokerLoadDataStore();
             for (int i = 0; i < loads.length; i++) {
                 loads[i] = loadStore.get("broker" + i).get().getWeightedMaxEMA();
@@ -785,16 +1111,16 @@ public void testLoadStats() {
             int i = 0;
             int j = loads.length - 1;
             Arrays.sort(loads);
-            for (int k = 0; k < conf.getLoadBalancerMaxNumberOfBrokerTransfersPerCycle(); k++) {
+            for (int k = 0; k < conf.getLoadBalancerMaxNumberOfBrokerSheddingPerCycle(); k++) {
                 double minLoad = loads[i];
                 double maxLoad = loads[j];
                 double offload = (maxLoad - minLoad) / 2;
                 Mean mean = new Mean();
                 StandardDeviation std = new StandardDeviation(false);
                 assertEquals(minLoad,
-                        loadStore.get(stats.minBrokers().pollLast()).get().getWeightedMaxEMA());
+                        loadStore.get(stats.peekMinBroker()).get().getWeightedMaxEMA());
                 assertEquals(maxLoad,
-                        loadStore.get(stats.maxBrokers().pollLast()).get().getWeightedMaxEMA());
+                        loadStore.get(stats.pollMaxBroker()).get().getWeightedMaxEMA());
                 assertEquals(stats.totalBrokers(), numBrokers);
                 assertEquals(stats.avg(), mean.evaluate(loads), delta);
                 assertEquals(stats.std(), std.evaluate(loads), delta);
@@ -804,4 +1130,41 @@ public void testLoadStats() {
             }
         }
     }
+
+    @Test
+    public void testHighVarianceLoadStats() {
+        int[] loads = {1, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100};
+        var ctx = getContext();
+        TransferShedder.LoadStats stats = new TransferShedder.LoadStats();
+        var loadStore = ctx.brokerLoadDataStore();
+        stats.setLoadDataStore(loadStore);
+        var conf = ctx.brokerConfiguration();
+        final Map availableBrokers = new HashMap<>();
+        for (int i = 0; i < loads.length; i++) {
+            availableBrokers.put("broker" + i, mock(BrokerLookupData.class));
+            loadStore.pushAsync("broker" + i, getCpuLoad(ctx,  loads[i], "broker" + i));
+        }
+        stats.update(loadStore, availableBrokers, Map.of(), conf);
+
+        assertEquals(stats.avg(), 0.9417647058823528);
+        assertEquals(stats.std(), 0.23294117647058868);
+    }
+
+    @Test
+    public void testLowVarianceLoadStats() {
+        int[] loads = {390, 391, 392, 393, 394, 395, 396, 397, 398, 399};
+        var ctx = getContext();
+        TransferShedder.LoadStats stats = new TransferShedder.LoadStats();
+        var loadStore = ctx.brokerLoadDataStore();
+        stats.setLoadDataStore(loadStore);
+        var conf = ctx.brokerConfiguration();
+        final Map availableBrokers = new HashMap<>();
+        for (int i = 0; i < loads.length; i++) {
+            availableBrokers.put("broker" + i, mock(BrokerLookupData.class));
+            loadStore.pushAsync("broker" + i, getCpuLoad(ctx,  loads[i], "broker" + i));
+        }
+        stats.update(loadStore, availableBrokers, Map.of(), conf);
+        assertEquals(stats.avg(), 3.9449999999999994);
+        assertEquals(stats.std(), 0.028722813232795824);
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
index 7d9cf556360e4..38d4e9904e649 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
@@ -54,8 +54,8 @@
 
 @Test(groups = "broker")
 public class UnloadSchedulerTest {
-
-   private ScheduledExecutorService loadManagerExecutor;
+    private PulsarService pulsar;
+    private ScheduledExecutorService loadManagerExecutor;
 
     public LoadManagerContext setupContext(){
         var ctx = getContext();
@@ -65,8 +65,12 @@ public LoadManagerContext setupContext(){
 
     @BeforeMethod
     public void setUp() {
-        this.loadManagerExecutor = Executors
-                .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager"));
+        this.pulsar = mock(PulsarService.class);
+        loadManagerExecutor = Executors
+                .newSingleThreadScheduledExecutor(new
+                        ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager"));
+        doReturn(loadManagerExecutor)
+                .when(pulsar).getLoadManagerExecutor();
     }
 
     @AfterMethod
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
index deff6fb00fafb..0aa055b58acd3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -31,10 +31,15 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
@@ -50,6 +55,9 @@
 public class DefaultNamespaceBundleSplitStrategyTest {
 
     PulsarService pulsar;
+    ExtensibleLoadManagerWrapper loadManagerWrapper;
+    ExtensibleLoadManagerImpl loadManager;
+    ServiceUnitStateChannel channel;
     BrokerService brokerService;
     PulsarStats pulsarStats;
     Map bundleStats;
@@ -76,7 +84,7 @@ void setup() {
         config.setLoadBalancerNamespaceBundleMaxMsgRate(100);
         config.setLoadBalancerNamespaceBundleMaxBandwidthMbytes(100);
         config.setLoadBalancerMaxNumberOfBundlesToSplitPerCycle(1);
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(3);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(3);
 
         pulsar = mock(PulsarService.class);
         brokerService = mock(BrokerService.class);
@@ -85,8 +93,9 @@ void setup() {
         namespaceBundleFactory = mock(NamespaceBundleFactory.class);
         loadManagerContext = mock(LoadManagerContext.class);
         brokerRegistry = mock(BrokerRegistry.class);
-
-
+        loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class);
+        loadManager = mock(ExtensibleLoadManagerImpl.class);
+        channel = mock(ServiceUnitStateChannelImpl.class);
 
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
@@ -96,6 +105,10 @@ void setup() {
         doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
         doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry();
         doReturn(broker).when(brokerRegistry).getBrokerId();
+        doReturn(new AtomicReference(loadManagerWrapper)).when(pulsar).getLoadManager();
+        doReturn(loadManager).when(loadManagerWrapper).get();
+        doReturn(channel).when(loadManager).getServiceUnitStateChannel();
+        doReturn(true).when(channel).isOwner(any());
 
 
         bundleStats = new LinkedHashMap<>();
@@ -109,7 +122,7 @@ void setup() {
     }
 
     public void testNamespaceBundleSplitConditionThreshold() {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
@@ -118,7 +131,7 @@ public void testNamespaceBundleSplitConditionThreshold() {
 
 
     public void testNotEnoughTopics() {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         bundleStats.values().forEach(v -> v.topics = 1);
@@ -128,7 +141,7 @@ public void testNotEnoughTopics() {
     }
 
     public void testNamespaceMaximumBundles() throws Exception {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         doReturn(config.getLoadBalancerNamespaceMaximumBundles()).when(namespaceService).getBundleCount(any());
@@ -138,7 +151,7 @@ public void testNamespaceMaximumBundles() throws Exception {
     }
 
     public void testEmptyBundleStats() {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         bundleStats.clear();
@@ -147,9 +160,21 @@ public void testEmptyBundleStats() {
         assertEquals(actual, expected);
     }
 
+    public void testNoBundleOwner() {
+        var counter = spy(new SplitCounter());
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        doReturn(false).when(channel).isOwner(any());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+        verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+    }
+
     public void testError() throws Exception {
         var counter = spy(new SplitCounter());
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
         doThrow(new RuntimeException()).when(namespaceService).getBundleCount(any());
@@ -162,7 +187,7 @@ public void testError() throws Exception {
     public void testMaxMsgRate() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> {
             v.msgRateOut = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
             v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
@@ -193,7 +218,7 @@ public void testMaxMsgRate() {
     public void testMaxTopics() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> v.topics = config.getLoadBalancerNamespaceBundleMaxTopics() + 1);
         for (int i = 0; i < threshold + 2; i++) {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
@@ -221,7 +246,7 @@ public void testMaxTopics() {
     public void testMaxSessions() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> {
             v.producerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
             v.consumerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
@@ -252,7 +277,7 @@ public void testMaxSessions() {
     public void testMaxBandwidthMbytes() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> {
             v.msgThroughputOut = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
             v.msgThroughputIn = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java
index dc1cdd3d5cbce..cd364990842a1 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java
@@ -20,10 +20,12 @@
 
 import java.io.Serializable;
 import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 /**
  */
 @EqualsAndHashCode
+@ToString
 public class NamespaceBundleStats implements Comparable, Serializable {
 
     public double msgRateIn;
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java
index eed8b33b39178..ae12e931071bc 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java
@@ -19,11 +19,13 @@
 package org.apache.pulsar.policies.data.loadbalancer;
 
 import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 /**
  * POJO used to represent any system specific resource usage this is the format that load manager expects it in.
  */
 @EqualsAndHashCode
+@ToString
 public class ResourceUsage {
     public final double usage;
     public final double limit;

From 38485e09ce1b01803bd6d5ab95901f6284428685 Mon Sep 17 00:00:00 2001
From: tison 
Date: Wed, 29 Mar 2023 17:55:48 +0800
Subject: [PATCH 124/404] [improve][build] Create source jar for
 pulsar-client-all shaded jar (#19956)

Signed-off-by: tison 
---
 pulsar-client-all/pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index 00da6e4895097..73621954e1fb8 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -141,6 +141,8 @@
               shade
             
             
+              true
+              true
               true
               true
               false

From 07acdbc8541c1eeb724361713e7fd136d4c93fc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Wed, 29 Mar 2023 14:38:02 +0200
Subject: [PATCH 125/404] [fix][sec] Fix transitive critical CVEs in
 file-system tiered storage (#19957)

---
 pom.xml                            |  6 +++---
 tiered-storage/file-system/pom.xml | 25 -------------------------
 2 files changed, 3 insertions(+), 28 deletions(-)

diff --git a/pom.xml b/pom.xml
index ac640f1a6a05f..4c489355281ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -180,8 +180,8 @@ flexible messaging model and an intuitive client API.
     0.3.2-patch11
     2.7.5
     0.4.4-hotfix1
-    3.3.3
-    2.4.7
+    3.3.5
+    2.4.10
     1.2.4
     8.5.2
     363
@@ -257,7 +257,7 @@ flexible messaging model and an intuitive client API.
     3.1
     4.2.0
     1.2.22
-    1.5.3
+    1.5.4
     5.4.0
     2.33.2
 
diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml
index 1d8aab48cbff8..745da1a95c70d 100644
--- a/tiered-storage/file-system/pom.xml
+++ b/tiered-storage/file-system/pom.xml
@@ -53,31 +53,6 @@
             
         
         
-        
-            com.sun.jersey
-            jersey-json
-            
-            1.19
-            
-                
-                    org.codehaus.jackson
-                    jackson-core-asl
-                
-                
-                    org.codehaus.jackson
-                    jackson-mapper-asl
-                
-                
-                    org.codehaus.jackson
-                    jackson-jaxrs
-                
-                
-                    org.codehaus.jackson
-                    jackson-xc
-                
-            
-        
-        
         
             org.apache.avro
             avro

From 55523ac8f31fd6d54aacba326edef1f53028877e Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com>
Date: Wed, 29 Mar 2023 13:55:22 -0700
Subject: [PATCH 126/404] [improve][io] KCA: flag to force optional primitive
 schemas (#19951)

Motivation
Kafka's schema has "Optional" flag that used there to validate data/allow nulls.
Pulsar's schema does not have such info which makes conversion to kafka schema lossy.

Modifications
Added a config parameter that lets one force primitive schemas into optional ones.
KV schema is always optional.

Default is false, to match existing behavior.
---
 .../io/kafka/connect/KafkaConnectSink.java    |  16 +-
 .../connect/PulsarKafkaConnectSinkConfig.java |   6 +
 .../schema/PulsarSchemaToKafkaSchema.java     |  55 ++++++-
 .../kafka/connect/KafkaConnectSinkTest.java   |  41 +++--
 .../PulsarSchemaToKafkaSchemaTest.java        | 149 +++++++++++++-----
 5 files changed, 203 insertions(+), 64 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index 475724cc4e545..10efc91ccdaad 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -90,6 +90,7 @@ public class KafkaConnectSink implements Sink {
     private PulsarKafkaConnectSinkConfig kafkaSinkConfig;
 
     protected String topicName;
+    protected boolean useOptionalPrimitives;
 
     private boolean sanitizeTopicName = false;
     // Thi is a workaround for https://github.com/apache/pulsar/issues/19922
@@ -164,6 +165,7 @@ public void open(Map config, SinkContext ctx) throws Exception {
         unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable();
         sanitizeTopicName = kafkaSinkConfig.isSanitizeTopicName();
         collapsePartitionedTopics = kafkaSinkConfig.isCollapsePartitionedTopics();
+        useOptionalPrimitives = kafkaSinkConfig.isUseOptionalPrimitives();
 
         useIndexAsOffset = kafkaSinkConfig.isUseIndexAsOffset();
         maxBatchBitsForOffset = kafkaSinkConfig.getMaxBatchBitsForOffset();
@@ -446,8 +448,11 @@ protected SinkRecord toSinkRecord(Record sourceRecord) {
                 && sourceRecord.getSchema().getSchemaInfo() != null
                 && sourceRecord.getSchema().getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
             KeyValueSchema kvSchema = (KeyValueSchema) sourceRecord.getSchema();
-            keySchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(kvSchema.getKeySchema());
-            valueSchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(kvSchema.getValueSchema());
+            // Assume Key_Value schema's key and value are always optional
+            keySchema = PulsarSchemaToKafkaSchema
+                    .getOptionalKafkaConnectSchema(kvSchema.getKeySchema(), useOptionalPrimitives);
+            valueSchema = PulsarSchemaToKafkaSchema
+                    .getOptionalKafkaConnectSchema(kvSchema.getValueSchema(), useOptionalPrimitives);
 
             Object nativeObject = sourceRecord.getValue().getNativeObject();
 
@@ -464,12 +469,13 @@ protected SinkRecord toSinkRecord(Record sourceRecord) {
         } else {
             if (sourceRecord.getMessage().get().hasBase64EncodedKey()) {
                 key = sourceRecord.getMessage().get().getKeyBytes();
-                keySchema = Schema.BYTES_SCHEMA;
+                keySchema = useOptionalPrimitives ? Schema.OPTIONAL_BYTES_SCHEMA : Schema.BYTES_SCHEMA;
             } else {
                 key = sourceRecord.getKey().orElse(null);
-                keySchema = Schema.STRING_SCHEMA;
+                keySchema = useOptionalPrimitives ? Schema.OPTIONAL_STRING_SCHEMA : Schema.STRING_SCHEMA;
             }
-            valueSchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(sourceRecord.getSchema());
+            valueSchema = PulsarSchemaToKafkaSchema
+                    .getKafkaConnectSchema(sourceRecord.getSchema(), useOptionalPrimitives);
             value = KafkaConnectData.getKafkaConnectData(sourceRecord.getValue().getNativeObject(), valueSchema);
         }
 
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
index 2525081a41ebb..96519e63e0afa 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
@@ -99,6 +99,12 @@ public class PulsarKafkaConnectSinkConfig implements Serializable {
             help = "Supply kafka record with topic name without -partition- suffix for partitioned topics.")
     private boolean collapsePartitionedTopics = false;
 
+    @FieldDoc(
+            defaultValue = "false",
+            help = "Pulsar schema does not contain information whether the Schema is optional, Kafka's does. \n"
+                    + "This provides a way to force all primitive schemas to be optional for Kafka. \n")
+    private boolean useOptionalPrimitives = false;
+
     public static PulsarKafkaConnectSinkConfig load(String yamlFile) throws IOException {
         ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
         return mapper.readValue(new File(yamlFile), PulsarKafkaConnectSinkConfig.class);
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
index faf28585e8aed..21a0a0f42e025 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
@@ -115,11 +115,15 @@ public Schema schema() {
     }
 
     private static final ImmutableMap pulsarSchemaTypeToKafkaSchema;
+    private static final ImmutableMap pulsarSchemaTypeToOptionalKafkaSchema;
     private static final ImmutableSet kafkaLogicalSchemas;
     private static final AvroData avroData = new AvroData(1000);
     private static final Cache schemaCache =
             CacheBuilder.newBuilder().maximumSize(10000)
                     .expireAfterAccess(30, TimeUnit.MINUTES).build();
+    private static final Cache optionalSchemaCache =
+            CacheBuilder.newBuilder().maximumSize(1000)
+                    .expireAfterAccess(30, TimeUnit.MINUTES).build();
 
     static {
         pulsarSchemaTypeToKafkaSchema = ImmutableMap.builder()
@@ -134,6 +138,17 @@ public Schema schema() {
                 .put(SchemaType.BYTES, Schema.BYTES_SCHEMA)
                 .put(SchemaType.DATE, Date.SCHEMA)
                 .build();
+        pulsarSchemaTypeToOptionalKafkaSchema = ImmutableMap.builder()
+                .put(SchemaType.BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA)
+                .put(SchemaType.INT8, Schema.OPTIONAL_INT8_SCHEMA)
+                .put(SchemaType.INT16, Schema.OPTIONAL_INT16_SCHEMA)
+                .put(SchemaType.INT32, Schema.OPTIONAL_INT32_SCHEMA)
+                .put(SchemaType.INT64, Schema.OPTIONAL_INT64_SCHEMA)
+                .put(SchemaType.FLOAT, Schema.OPTIONAL_FLOAT32_SCHEMA)
+                .put(SchemaType.DOUBLE, Schema.OPTIONAL_FLOAT64_SCHEMA)
+                .put(SchemaType.STRING, Schema.OPTIONAL_STRING_SCHEMA)
+                .put(SchemaType.BYTES, Schema.OPTIONAL_BYTES_SCHEMA)
+                .build();
         kafkaLogicalSchemas = ImmutableSet.builder()
                 .add(Timestamp.LOGICAL_NAME)
                 .add(Date.LOGICAL_NAME)
@@ -153,12 +168,33 @@ private static org.apache.avro.Schema parseAvroSchema(String schemaJson) {
         return parser.parse(schemaJson);
     }
 
-    public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
-        Schema s = getKafkaConnectSchema(pulsarSchema);
-        return new OptionalForcingSchema(s);
+    public static Schema makeOptional(Schema s) {
+        if (s == null || s.isOptional()) {
+            return s;
+        }
+
+        String logicalSchemaName = s.name();
+        if (kafkaLogicalSchemas.contains(logicalSchemaName)) {
+            return s;
+        }
+
+        try {
+            return optionalSchemaCache.get(s, () -> new OptionalForcingSchema(s));
+        } catch (ExecutionException | UncheckedExecutionException | ExecutionError ee) {
+            String msg = "Failed to create optional schema for " + s;
+            log.error(msg);
+            throw new IllegalStateException(msg, ee);
+        }
     }
 
-    public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
+    public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema,
+                                               boolean useOptionalPrimitives) {
+        return makeOptional(getKafkaConnectSchema(pulsarSchema, useOptionalPrimitives));
+
+    }
+
+    public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema,
+                                               boolean useOptionalPrimitives) {
         if (pulsarSchema == null || pulsarSchema.getSchemaInfo() == null) {
             throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null);
         }
@@ -191,6 +227,11 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p
             throw new IllegalStateException("Unsupported Kafka Logical Schema " + logicalSchemaName);
         }
 
+        if (useOptionalPrimitives
+                && pulsarSchemaTypeToOptionalKafkaSchema.containsKey(pulsarSchema.getSchemaInfo().getType())) {
+            return pulsarSchemaTypeToOptionalKafkaSchema.get(pulsarSchema.getSchemaInfo().getType());
+        }
+
         if (pulsarSchemaTypeToKafkaSchema.containsKey(pulsarSchema.getSchemaInfo().getType())) {
             return pulsarSchemaTypeToKafkaSchema.get(pulsarSchema.getSchemaInfo().getType());
         }
@@ -199,8 +240,10 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p
             return schemaCache.get(pulsarSchema.getSchemaInfo().getSchema(), () -> {
                 if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
                     KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
-                    return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()),
-                                    getOptionalKafkaConnectSchema(kvSchema.getValueSchema()))
+                    return SchemaBuilder.map(
+                            makeOptional(getKafkaConnectSchema(kvSchema.getKeySchema(), useOptionalPrimitives)),
+                            makeOptional(getKafkaConnectSchema(kvSchema.getValueSchema(), useOptionalPrimitives)))
+                                .optional()
                                 .build();
                 }
                 org.apache.avro.Schema avroSchema = parseAvroSchema(
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index 6ccfa3b71a2f7..5410e0bb8d664 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -162,7 +162,7 @@ public T answer(InvocationOnMock invocationOnMock) throws Throwable {
         }
     }
 
-    private String offsetTopicName = "persistent://my-property/my-ns/kafka-connect-sink-offset";
+    final private String offsetTopicName = "persistent://my-property/my-ns/kafka-connect-sink-offset";
 
     private Path file;
     private Map props;
@@ -797,7 +797,9 @@ public void kafkaLogicalTypesTimestampTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         java.util.Date date = getDateFromString("12/30/1999 11:12:13");
         Object connectData = KafkaConnectData
@@ -815,7 +817,9 @@ public void kafkaLogicalTypesTimeTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         java.util.Date date = getDateFromString("01/01/1970 11:12:13");
         Object connectData = KafkaConnectData
@@ -833,7 +837,9 @@ public void kafkaLogicalTypesDateTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         java.util.Date date = getDateFromString("12/31/2022 00:00:00");
         Object connectData = KafkaConnectData
@@ -854,7 +860,9 @@ public void kafkaLogicalTypesDecimalTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         Object connectData = KafkaConnectData
                 .getKafkaConnectData(Decimal.fromLogical(kafkaSchema, BigDecimal.valueOf(100L, 10)), kafkaSchema);
@@ -874,11 +882,11 @@ public void connectDataComplexAvroSchemaGenericRecordTest() {
                 getGenericRecord(value, pulsarAvroSchema));
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(Schema.KeyValue(pulsarAvroSchema, pulsarAvroSchema));
+                .getKafkaConnectSchema(Schema.KeyValue(pulsarAvroSchema, pulsarAvroSchema), false);
 
-        Object connectData = KafkaConnectData.getKafkaConnectData(kv, kafkaSchema);
-
-        org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
+        Assert.assertTrue(kafkaSchema.isOptional());
+        Assert.assertTrue(kafkaSchema.keySchema().isOptional());
+        Assert.assertTrue(kafkaSchema.valueSchema().isOptional());
     }
 
     @Test
@@ -990,7 +998,8 @@ private void testPojoAsAvroAndJsonConversionToConnectData(Object pojo, AvroSchem
         Object value = pojoAsAvroRecord(pojo, pulsarAvroSchema);
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(pulsarAvroSchema);
+                .getKafkaConnectSchema(pulsarAvroSchema, false);
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         Object connectData = KafkaConnectData.getKafkaConnectData(value, kafkaSchema);
 
@@ -999,6 +1008,18 @@ private void testPojoAsAvroAndJsonConversionToConnectData(Object pojo, AvroSchem
         Object jsonNode = pojoAsJsonNode(pojo);
         connectData = KafkaConnectData.getKafkaConnectData(jsonNode, kafkaSchema);
         org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
+
+        kafkaSchema = PulsarSchemaToKafkaSchema
+                .getKafkaConnectSchema(pulsarAvroSchema, true);
+        Assert.assertFalse(kafkaSchema.isOptional());
+
+        connectData = KafkaConnectData.getKafkaConnectData(value, kafkaSchema);
+
+        org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
+
+        jsonNode = pojoAsJsonNode(pojo);
+        connectData = KafkaConnectData.getKafkaConnectData(jsonNode, kafkaSchema);
+        org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
     }
 
     private JsonNode pojoAsJsonNode(Object pojo) {
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java
index 9cc6db034c870..b236365bbb8a1 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java
@@ -32,6 +32,7 @@
 import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl;
 import org.apache.pulsar.io.kafka.connect.schema.KafkaConnectData;
 import org.apache.pulsar.io.kafka.connect.schema.PulsarSchemaToKafkaSchema;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.math.BigInteger;
@@ -39,6 +40,8 @@
 import java.util.Map;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
 
 /**
  * Test the conversion of PulsarSchema To KafkaSchema\.
@@ -132,101 +135,134 @@ static class ComplexStruct {
         String[] stringArr;
     }
 
-    @Test
-    public void bytesSchemaTest() {
+    @DataProvider(name = "useOptionalPrimitives")
+    public static Object[][] useOptionalPrimitives() {
+        return new Object[][] {
+                {true},
+                {false}
+        };
+    }
+
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void bytesSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTES);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTES, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
 
         kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTEBUFFER);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTEBUFFER, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void stringSchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void stringSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.STRING);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.STRING, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRING);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void booleanSchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void booleanSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BOOL);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BOOL, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BOOLEAN);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int8SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int8SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT8);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT8, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT8);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int16SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int16SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT16);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT16, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT16);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int32SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int32SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT32);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT32, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT32);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int64SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int64SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT64);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT64, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT64);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void float32SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void float32SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.FLOAT);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.FLOAT, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.FLOAT32);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void float64SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void float64SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DOUBLE);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DOUBLE, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.FLOAT64);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void kvBytesSchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void kvBytesSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.KV_BYTES());
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.KV_BYTES(), useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.MAP);
         assertEquals(kafkaSchema.keySchema().type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
         assertEquals(kafkaSchema.valueSchema().type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
+        assertTrue(kafkaSchema.isOptional());
+
+        // key and value are always optional
+        assertTrue(kafkaSchema.keySchema().isOptional());
+        assertTrue(kafkaSchema.valueSchema().isOptional());
     }
 
     @Test
     public void kvBytesIntSchemaTests() {
         Schema pulsarKvSchema = KeyValueSchemaImpl.of(Schema.STRING, Schema.INT64);
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarKvSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarKvSchema, false);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.MAP);
         assertEquals(kafkaSchema.keySchema().type(), org.apache.kafka.connect.data.Schema.Type.STRING);
         assertEquals(kafkaSchema.valueSchema().type(), org.apache.kafka.connect.data.Schema.Type.INT64);
+        assertTrue(kafkaSchema.isOptional());
+
+        // key and value are always optional
+        assertTrue(kafkaSchema.keySchema().isOptional());
+        assertTrue(kafkaSchema.valueSchema().isOptional());
     }
 
     @Test
     public void avroSchemaTest() {
         AvroSchema pulsarAvroSchema = AvroSchema.of(StructWithAnnotations.class);
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, false);
+        org.apache.kafka.connect.data.Schema kafkaSchemaOpt =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, true);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), STRUCT_FIELDS.size());
         for (String name: STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            // set by avro schema
+            assertEquals(kafkaSchema.field(name).schema().isOptional(),
+                    kafkaSchemaOpt.field(name).schema().isOptional());
         }
     }
 
@@ -234,11 +270,16 @@ public void avroSchemaTest() {
     public void avroComplexSchemaTest() {
         AvroSchema pulsarAvroSchema = AvroSchema.of(ComplexStruct.class);
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, false);
+        org.apache.kafka.connect.data.Schema kafkaSchemaOpt =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, true);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size());
         for (String name: COMPLEX_STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            // set by avro schema
+            assertEquals(kafkaSchema.field(name).schema().isOptional(),
+                    kafkaSchemaOpt.field(name).schema().isOptional());
         }
     }
 
@@ -250,11 +291,16 @@ public void jsonSchemaTest() {
                 .withAlwaysAllowNull(false)
                 .build());
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, false);
+        org.apache.kafka.connect.data.Schema kafkaSchemaOpt =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, true);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), STRUCT_FIELDS.size());
         for (String name: STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            // set by schema
+            assertEquals(kafkaSchema.field(name).schema().isOptional(),
+                    kafkaSchemaOpt.field(name).schema().isOptional());
         }
     }
 
@@ -266,11 +312,27 @@ public void jsonComplexSchemaTest() {
                         .withAlwaysAllowNull(false)
                         .build());
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, false);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size());
         for (String name: COMPLEX_STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            assertFalse(kafkaSchema.field(name).schema().isOptional());
+        }
+
+        kafkaSchema =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, true);
+        assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
+        assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size());
+        for (String name: COMPLEX_STRUCT_FIELDS) {
+            assertEquals(kafkaSchema.field(name).name(), name);
+            assertFalse(kafkaSchema.field(name).schema().isOptional());
+
+            if (kafkaSchema.field(name).schema().type().isPrimitive()) {
+                // false because .withAlwaysAllowNull(false), avroschema values are used
+                assertFalse(kafkaSchema.field(name).schema().isOptional(),
+                        kafkaSchema.field(name).schema().type().getName());
+            }
         }
     }
 
@@ -308,39 +370,40 @@ public void castToKafkaSchemaTest() {
     @Test
     public void dateSchemaTest() {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DATE);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DATE, true);
         assertEquals(kafkaSchema.type(), Date.SCHEMA.type());
+        assertFalse(kafkaSchema.isOptional());
     }
 
     // not supported schemas below:
     @Test(expectedExceptions = IllegalStateException.class)
     public void timeSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIME);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIME, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void timestampSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIMESTAMP);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIMESTAMP, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void instantSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INSTANT);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INSTANT, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void localDateSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void localTimeSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_TIME);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_TIME, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void localDatetimeSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE_TIME);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE_TIME, false);
     }
 
 }

From 68c10eed7604aa3dcc3a6d8b548575e99b94dca2 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Thu, 30 Mar 2023 16:32:13 +0800
Subject: [PATCH 127/404] [feat][broker][PIP-195] Add metrics for bucket
 delayed message tracker (#19716)

---
 .../pulsar/broker/ServiceConfiguration.java   |   2 +-
 .../delayed/DelayedDeliveryTracker.java       |   6 -
 .../InMemoryDelayedDeliveryTracker.java       |   5 -
 .../BookkeeperBucketSnapshotStorage.java      |  15 +-
 .../bucket/BucketDelayedDeliveryTracker.java  |  74 +++++++--
 .../BucketDelayedMessageIndexStats.java       | 146 ++++++++++++++++++
 .../delayed/bucket/ImmutableBucket.java       |  28 +++-
 .../broker/delayed/bucket/MutableBucket.java  |   6 +-
 ...PersistentDispatcherMultipleConsumers.java |  15 +-
 .../persistent/PersistentSubscription.java    |   3 +
 .../service/persistent/PersistentTopic.java   |   9 ++
 .../prometheus/AggregatedNamespaceStats.java  |  19 +++
 .../AggregatedSubscriptionStats.java          |   3 +
 .../prometheus/NamespaceStatsAggregator.java  |  13 +-
 .../broker/stats/prometheus/TopicStats.java   |  22 +++
 .../delayed/MockBucketSnapshotStorage.java    |   2 +-
 .../BucketDelayedDeliveryTrackerTest.java     |  23 +--
 .../persistent/BucketDelayedDeliveryTest.java | 143 +++++++++++++++++
 .../data/stats/SubscriptionStatsImpl.java     |  12 ++
 .../policies/data/stats/TopicMetricBean.java  |  30 ++++
 .../policies/data/stats/TopicStatsImpl.java   |  13 ++
 21 files changed, 532 insertions(+), 57 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java
 create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 03e0fec0a5346..52cebe15f6a5c 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -366,7 +366,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
-            The max number of delayed message index in per bucket snapshot segment, -1 means no limitation\
+            The max number of delayed message index in per bucket snapshot segment, -1 means no limitation, \
             after reaching the max number limitation, the snapshot segment will be cut off.""")
     private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment = 5000;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
index 3cc2da8db1e4d..78229fef25a5a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
@@ -67,12 +67,6 @@ public interface DelayedDeliveryTracker extends AutoCloseable {
      */
     boolean shouldPauseAllDeliveries();
 
-    /**
-     * Tells whether this DelayedDeliveryTracker contains this message index,
-     * if the tracker is not supported it or disabled this feature also will return false.
-     */
-    boolean containsMessage(long ledgerId, long entryId);
-
     /**
      *  Reset tick time use zk policies cache.
      * @param tickTime
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
index 8de6ee58e2ce5..58358b06a46bb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
@@ -178,11 +178,6 @@ && getNumberOfDelayedMessages() >= fixedDelayDetectionLookahead
                 && !hasMessageAvailable();
     }
 
-    @Override
-    public boolean containsMessage(long ledgerId, long entryId) {
-        return false;
-    }
-
     protected long nextDeliveryTime() {
         return priorityQueue.peekN1();
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 08202bb19155d..e7d4f9301dd36 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -67,7 +67,7 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet
     @Override
     public CompletableFuture getBucketSnapshotMetadata(long bucketId) {
         return openLedger(bucketId).thenCompose(
-                ledgerHandle -> getLedgerEntryThenCloseLedger(ledgerHandle, 0, 0).
+                ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0).
                         thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())));
     }
 
@@ -75,17 +75,13 @@ public CompletableFuture getBucketSnapshotMetadata(long bucket
     public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId,
                                                                              long lastSegmentEntryId) {
         return openLedger(bucketId).thenCompose(
-                ledgerHandle -> getLedgerEntryThenCloseLedger(ledgerHandle, firstSegmentEntryId,
+                ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId,
                         lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries));
     }
 
     @Override
     public CompletableFuture getBucketSnapshotLength(long bucketId) {
-        return openLedger(bucketId).thenApply(ledgerHandle -> {
-            long length = ledgerHandle.getLength();
-            closeLedger(ledgerHandle);
-            return length;
-        });
+        return openLedger(bucketId).thenApply(LedgerHandle::getLength);
     }
 
     @Override
@@ -212,8 +208,8 @@ private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data)
         });
     }
 
-    CompletableFuture> getLedgerEntryThenCloseLedger(LedgerHandle ledger,
-                                                                              long firstEntryId, long lastEntryId) {
+    CompletableFuture> getLedgerEntry(LedgerHandle ledger,
+                                                               long firstEntryId, long lastEntryId) {
         final CompletableFuture> future = new CompletableFuture<>();
         ledger.asyncReadEntries(firstEntryId, lastEntryId,
                 (rc, handle, entries, ctx) -> {
@@ -222,7 +218,6 @@ CompletableFuture> getLedgerEntryThenCloseLedger(Ledger
                     } else {
                         future.complete(entries);
                     }
-                    closeLedger(handle);
                 }, null
         );
         return future;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index ef7be187cec3d..a34bd51af98e4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -57,6 +57,7 @@
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 import org.roaringbitmap.RoaringBitmap;
@@ -69,6 +70,10 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     static final int AsyncOperationTimeoutSeconds = 60;
 
+    private static final Long INVALID_BUCKET_ID = -1L;
+
+    private static final int MAX_MERGE_NUM = 4;
+
     private final long minIndexCountPerBucket;
 
     private final long timeStepPerBucketSnapshotSegmentInMillis;
@@ -93,9 +98,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final Table snapshotSegmentLastIndexTable;
 
-    private static final Long INVALID_BUCKET_ID = -1L;
-
-    private static final int MAX_MERGE_NUM = 4;
+    private final BucketDelayedMessageIndexStats stats;
 
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
@@ -125,6 +128,7 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
         this.lastMutableBucket =
                 new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(),
                         bucketSnapshotStorage);
+        this.stats = new BucketDelayedMessageIndexStats();
         this.numberDelayedMessages = recoverBucketSnapshot();
     }
 
@@ -161,8 +165,9 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException {
         }
 
         try {
-            FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+            FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 2, TimeUnit.SECONDS);
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
+            log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e);
             if (e instanceof InterruptedException) {
                 Thread.currentThread().interrupt();
             }
@@ -193,7 +198,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException {
             ImmutableBucket immutableBucket = mapEntry.getValue();
             immutableBucketMap.remove(key);
             // delete asynchronously without waiting for completion
-            immutableBucket.asyncDeleteBucketSnapshot();
+            immutableBucket.asyncDeleteBucketSnapshot(stats);
         }
 
         MutableLong numberDelayedMessages = new MutableLong(0);
@@ -246,7 +251,8 @@ private Optional findImmutableBucket(long ledgerId) {
         return Optional.ofNullable(immutableBuckets.get(ledgerId));
     }
 
-    private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair) {
+    private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair,
+                                            long startTime) {
         if (immutableBucketDelayedIndexPair != null) {
             ImmutableBucket immutableBucket = immutableBucketDelayedIndexPair.getLeft();
             immutableBuckets.put(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId),
@@ -260,14 +266,19 @@ private void afterCreateImmutableBucket(Pair immu
                 CompletableFuture future = createFuture.handle((bucketId, ex) -> {
                     if (ex == null) {
                         immutableBucket.setSnapshotSegments(null);
+                        immutableBucket.asyncUpdateSnapshotLength();
                         log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(),
                                 immutableBucket.bucketKey());
+
+                        stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create,
+                                System.currentTimeMillis() - startTime);
+
                         return bucketId;
                     }
 
-                    //TODO Record create snapshot failed
-                    log.error("[{}] Failed to create bucket snapshot, bucketKey: {}",
-                            dispatcher.getName(), immutableBucket.bucketKey(), ex);
+                    log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", dispatcher.getName(),
+                            immutableBucket.bucketKey(), ex);
+                    stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.create);
 
                     // Put indexes back into the shared queue and downgrade to memory mode
                     synchronized (BucketDelayedDeliveryTracker.this) {
@@ -311,12 +322,14 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
         if (!existBucket && ledgerId > lastMutableBucket.endLedgerId
                 && lastMutableBucket.size() >= minIndexCountPerBucket
                 && !lastMutableBucket.isEmpty()) {
+            long createStartTime = System.currentTimeMillis();
+            stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.create);
             Pair immutableBucketDelayedIndexPair =
                     lastMutableBucket.sealBucketAndAsyncPersistent(
                             this.timeStepPerBucketSnapshotSegmentInMillis,
                             this.maxIndexesPerBucketSnapshotSegment,
                             this.sharedBucketPriorityQueue);
-            afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
+            afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime);
             lastMutableBucket.resetLastMutableBucketRange();
 
             if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) {
@@ -374,7 +387,7 @@ private synchronized List selectMergedBuckets(final List= 0) {
-            return values.subList(minIndex, minIndex + MAX_MERGE_NUM);
+            return values.subList(minIndex, minIndex + mergeNum);
         } else if (mergeNum > 2){
             return selectMergedBuckets(values, mergeNum - 1);
         } else {
@@ -400,6 +413,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
         for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
             immutableBucket.merging = true;
         }
+
+        long mergeStartTime = System.currentTimeMillis();
+        stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.merge);
         return asyncMergeBucketSnapshot(toBeMergeImmutableBuckets).whenComplete((__, ex) -> {
             synchronized (this) {
                 for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
@@ -409,9 +425,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
             if (ex != null) {
                 log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}",
                         dispatcher.getName(), bucketsStr, ex);
+
+                stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.merge);
             } else {
                 log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}",
                         dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size());
+
+                stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.merge,
+                        System.currentTimeMillis() - mergeStartTime);
             }
         });
     }
@@ -436,6 +457,8 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List {
                         synchronized (BucketDelayedDeliveryTracker.this) {
+                            long createStartTime = System.currentTimeMillis();
+                            stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.create);
                             Pair immutableBucketDelayedIndexPair =
                                     lastMutableBucket.createImmutableBucketAndAsyncPersistent(
                                             timeStepPerBucketSnapshotSegmentInMillis,
@@ -461,12 +484,12 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List {
                                         List> removeFutures =
-                                                buckets.stream().map(ImmutableBucket::asyncDeleteBucketSnapshot)
+                                                buckets.stream().map(bucket -> bucket.asyncDeleteBucketSnapshot(stats))
                                                         .toList();
                                         return FutureUtil.waitForAll(removeFutures);
                                     });
@@ -557,15 +580,17 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
                     if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) {
                         immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
-                        bucket.asyncDeleteBucketSnapshot();
+                        bucket.asyncDeleteBucketSnapshot(stats);
                         continue;
                     }
 
+                    long loadStartTime = System.currentTimeMillis();
+                    stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load);
                     bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> {
                         if (CollectionUtils.isEmpty(indexList)) {
                             immutableBuckets.asMapOfRanges()
                                     .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
-                            bucket.asyncDeleteBucketSnapshot();
+                            bucket.asyncDeleteBucketSnapshot(stats);
                             return;
                         }
                         DelayedMessageIndexBucketSnapshotFormat.DelayedIndex
@@ -583,9 +608,14 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
                             log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}",
                                     dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex);
+
+                            stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load);
                         } else {
                             log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}",
                                     dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1);
+
+                            stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load,
+                                    System.currentTimeMillis() - loadStartTime);
                         }
                     }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS);
                 } catch (Exception e) {
@@ -645,7 +675,7 @@ private CompletableFuture cleanImmutableBuckets() {
         Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
         while (iterator.hasNext()) {
             ImmutableBucket bucket = iterator.next();
-            futures.add(bucket.clear());
+            futures.add(bucket.clear(stats));
             numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
             iterator.remove();
         }
@@ -661,7 +691,6 @@ private boolean removeIndexBit(long ledgerId, long entryId) {
                 .orElse(false);
     }
 
-    @Override
     public boolean containsMessage(long ledgerId, long entryId) {
         if (lastMutableBucket.containsMessage(ledgerId, entryId)) {
             return true;
@@ -670,4 +699,15 @@ public boolean containsMessage(long ledgerId, long entryId) {
         return findImmutableBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, entryId))
                 .orElse(false);
     }
+
+    public Map genTopicMetricMap() {
+        stats.recordNumOfBuckets(immutableBuckets.asMapOfRanges().size() + 1);
+        stats.recordDelayedMessageIndexLoaded(this.sharedBucketPriorityQueue.size() + this.lastMutableBucket.size());
+        MutableLong totalSnapshotLength = new MutableLong();
+        immutableBuckets.asMapOfRanges().values().forEach(immutableBucket -> {
+            totalSnapshotLength.add(immutableBucket.getSnapshotLength());
+        });
+        stats.recordBucketSnapshotSizeBytes(totalSnapshotLength.longValue());
+        return stats.genTopicMetricMap();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java
new file mode 100644
index 0000000000000..68788c359d560
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.delayed.bucket;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.bookkeeper.mledger.util.StatsBuckets;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
+
+public class BucketDelayedMessageIndexStats {
+
+    private static final long[] BUCKETS = new long[]{50, 100, 500, 1000, 5000, 30000, 60000};
+
+    enum State {
+        succeed,
+        failed,
+        all
+    }
+
+    enum Type {
+        create,
+        load,
+        delete,
+        merge
+    }
+
+    private static final String BUCKET_TOTAL_NAME = "pulsar_delayed_message_index_bucket_total";
+    private static final String INDEX_LOADED_NAME = "pulsar_delayed_message_index_loaded";
+    private static final String SNAPSHOT_SIZE_BYTES_NAME = "pulsar_delayed_message_index_bucket_snapshot_size_bytes";
+    private static final String OP_COUNT_NAME = "pulsar_delayed_message_index_bucket_op_count";
+    private static final String OP_LATENCY_NAME = "pulsar_delayed_message_index_bucket_op_latency_ms";
+
+    private final AtomicInteger delayedMessageIndexBucketTotal = new AtomicInteger();
+    private final AtomicLong delayedMessageIndexLoaded = new AtomicLong();
+    private final AtomicLong delayedMessageIndexBucketSnapshotSizeBytes = new AtomicLong();
+    private final Map delayedMessageIndexBucketOpLatencyMs = new ConcurrentHashMap<>();
+    private final Map delayedMessageIndexBucketOpCount = new ConcurrentHashMap<>();
+
+    public BucketDelayedMessageIndexStats() {
+    }
+
+    public Map genTopicMetricMap() {
+        Map metrics = new HashMap<>();
+
+        metrics.put(BUCKET_TOTAL_NAME,
+                new TopicMetricBean(BUCKET_TOTAL_NAME, delayedMessageIndexBucketTotal.get(), null));
+
+        metrics.put(INDEX_LOADED_NAME,
+                new TopicMetricBean(INDEX_LOADED_NAME, delayedMessageIndexLoaded.get(), null));
+
+        metrics.put(SNAPSHOT_SIZE_BYTES_NAME,
+                new TopicMetricBean(SNAPSHOT_SIZE_BYTES_NAME, delayedMessageIndexBucketSnapshotSizeBytes.get(), null));
+
+        delayedMessageIndexBucketOpCount.forEach((k, count) -> {
+            String[] labels = splitKey(k);
+            String[] labelsAndValues = new String[] {"state", labels[0], "type", labels[1]};
+            String key = OP_COUNT_NAME + joinKey(labelsAndValues);
+            metrics.put(key, new TopicMetricBean(OP_COUNT_NAME, count.sumThenReset(), labelsAndValues));
+        });
+
+        delayedMessageIndexBucketOpLatencyMs.forEach((typeName, statsBuckets) -> {
+            statsBuckets.refresh();
+            long[] buckets = statsBuckets.getBuckets();
+            for (int i = 0; i < buckets.length; i++) {
+                long count = buckets[i];
+                if (count == 0L) {
+                    continue;
+                }
+                String quantile;
+                if (i == BUCKETS.length) {
+                    quantile = "overflow";
+                } else {
+                    quantile = String.valueOf(BUCKETS[i]);
+                }
+                String[] labelsAndValues = new String[] {"type", typeName, "quantile", quantile};
+                String key = OP_LATENCY_NAME + joinKey(labelsAndValues);
+
+                metrics.put(key, new TopicMetricBean(OP_LATENCY_NAME, count, labelsAndValues));
+            }
+            String[] labelsAndValues = new String[] {"type", typeName};
+            metrics.put(OP_LATENCY_NAME + "_count" + joinKey(labelsAndValues),
+                    new TopicMetricBean(OP_LATENCY_NAME + "_count", statsBuckets.getCount(), labelsAndValues));
+            metrics.put(OP_LATENCY_NAME + "_sum" + joinKey(labelsAndValues),
+                    new TopicMetricBean(OP_LATENCY_NAME + "_sum", statsBuckets.getSum(), labelsAndValues));
+        });
+
+        return metrics;
+    }
+
+    public void recordNumOfBuckets(int numOfBuckets) {
+        delayedMessageIndexBucketTotal.set(numOfBuckets);
+    }
+
+    public void recordDelayedMessageIndexLoaded(long num) {
+        delayedMessageIndexLoaded.set(num);
+    }
+
+    public void recordBucketSnapshotSizeBytes(long sizeBytes) {
+        delayedMessageIndexBucketSnapshotSizeBytes.set(sizeBytes);
+    }
+
+    public void recordTriggerEvent(Type eventType) {
+        delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.all.name(), eventType.name()),
+                k -> new LongAdder()).increment();
+    }
+
+    public void recordSuccessEvent(Type eventType, long cost) {
+        delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.succeed.name(), eventType.name()),
+                k -> new LongAdder()).increment();
+        delayedMessageIndexBucketOpLatencyMs.computeIfAbsent(eventType.name(),
+                k -> new StatsBuckets(BUCKETS)).addValue(cost);
+    }
+
+    public void recordFailEvent(Type eventType) {
+        delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.failed.name(), eventType.name()),
+                k -> new LongAdder()).increment();
+    }
+
+    public static String joinKey(String... values) {
+        return String.join("_", values);
+    }
+
+    public static String[] splitKey(String key) {
+        return key.split("_");
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index 969d326e28187..82e98cefa5d98 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -131,6 +131,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
                         List indexList =
                                 snapshotSegment.getIndexesList();
                         this.setCurrentSegmentEntryId(nextSegmentEntryId);
+                        if (isRecover) {
+                            this.asyncUpdateSnapshotLength();
+                        }
                         return indexList;
                     });
         });
@@ -175,7 +178,9 @@ CompletableFuture>
         }, BucketSnapshotPersistenceException.class, MaxRetryTimes);
     }
 
-    CompletableFuture asyncDeleteBucketSnapshot() {
+    CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats) {
+        long deleteStartTime = System.currentTimeMillis();
+        stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.delete);
         String bucketKey = bucketKey();
         long bucketId = getAndUpdateBucketId();
         return removeBucketCursorProperty(bucketKey).thenCompose(__ ->
@@ -184,16 +189,33 @@ CompletableFuture asyncDeleteBucketSnapshot() {
                     if (ex != null) {
                         log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
                                 dispatcherName, bucketId, bucketKey, ex);
+
+                        stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete);
                     } else {
                         log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
                                  dispatcherName, bucketId, bucketKey);
+
+                        stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete,
+                                System.currentTimeMillis() - deleteStartTime);
                     }
         });
     }
 
-    CompletableFuture clear() {
+    CompletableFuture clear(BucketDelayedMessageIndexStats stats) {
         delayedIndexBitMap.clear();
         return getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null)
-                .thenCompose(__ -> asyncDeleteBucketSnapshot());
+                .thenCompose(__ -> asyncDeleteBucketSnapshot(stats));
+    }
+
+    protected CompletableFuture asyncUpdateSnapshotLength() {
+        long bucketId = getAndUpdateBucketId();
+        return bucketSnapshotStorage.getBucketSnapshotLength(bucketId).whenComplete((length, ex) -> {
+            if (ex != null) {
+                log.error("[{}] Failed to get snapshot length, bucketId: {}, bucketKey: {}",
+                        dispatcherName, bucketId, bucketKey(), ex);
+            } else {
+                setSnapshotLength(length);
+            }
+        });
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index f8a4ecc7a4ddb..e49ebe9606e01 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -62,8 +62,10 @@ Pair createImmutableBucketAndAsyncPersistent(
             final long timeStepPerBucketSnapshotSegment, final int maxIndexesPerBucketSnapshotSegment,
             TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId,
             final long endLedgerId) {
-        log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName,
-                startLedgerId, endLedgerId);
+        if (log.isDebugEnabled()) {
+            log.debug("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName,
+                    startLedgerId, endLedgerId);
+        }
 
         if (delayedIndexQueue.isEmpty()) {
             return null;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index e5c9e85bac3f2..7ff6e72d02aed 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -72,6 +72,7 @@
 import org.apache.pulsar.client.impl.Backoff;
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.Codec;
 import org.slf4j.Logger;
@@ -332,7 +333,7 @@ public synchronized void readMoreEntries() {
                     Predicate skipCondition = null;
                     final DelayedDeliveryTracker deliveryTracker = delayedDeliveryTracker.get();
                     if (deliveryTracker instanceof BucketDelayedDeliveryTracker) {
-                        skipCondition = position -> deliveryTracker
+                        skipCondition = position -> ((BucketDelayedDeliveryTracker) deliveryTracker)
                                 .containsMessage(position.getLedgerId(), position.getEntryId());
                     }
                     cursor.asyncReadEntriesWithSkipOrWait(messagesToRead, bytesToRead, this, ReadType.Normal,
@@ -1180,6 +1181,18 @@ public long getDelayedTrackerMemoryUsage() {
         return 0;
     }
 
+    public Map getBucketDelayedIndexStats() {
+        if (delayedDeliveryTracker.isEmpty()) {
+            return Collections.emptyMap();
+        }
+
+        if (delayedDeliveryTracker.get() instanceof BucketDelayedDeliveryTracker) {
+            return ((BucketDelayedDeliveryTracker) delayedDeliveryTracker.get()).genTopicMetricMap();
+        }
+
+        return Collections.emptyMap();
+    }
+
     public ManagedCursor getCursor() {
         return cursor;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index bdb3c9fc391ce..4ed191a9b4f61 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -1147,6 +1147,9 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri
         if (dispatcher instanceof PersistentDispatcherMultipleConsumers) {
             subStats.delayedMessageIndexSizeInBytes =
                     ((PersistentDispatcherMultipleConsumers) dispatcher).getDelayedTrackerMemoryUsage();
+
+            subStats.bucketDelayedIndexStats =
+                    ((PersistentDispatcherMultipleConsumers) dispatcher).getBucketDelayedIndexStats();
         }
 
         if (Subscription.isIndividualAckMode(subType)) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 82a4f5312357a..fa08330ff3c35 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -153,6 +153,7 @@
 import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
 import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl;
 import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.schema.SchemaData;
@@ -2183,6 +2184,14 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog
             stats.nonContiguousDeletedMessagesRangesSerializedSize +=
                     subStats.nonContiguousDeletedMessagesRangesSerializedSize;
             stats.delayedMessageIndexSizeInBytes += subStats.delayedMessageIndexSizeInBytes;
+
+            subStats.bucketDelayedIndexStats.forEach((k, v) -> {
+                TopicMetricBean topicMetricBean =
+                        stats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+                topicMetricBean.name = v.name;
+                topicMetricBean.labelsAndValues = v.labelsAndValues;
+                topicMetricBean.value += v.value;
+            });
         });
 
         replicators.forEach((cluster, replicator) -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
index 0a905daa341f3..ea77bd69302a0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.bookkeeper.mledger.util.StatsBuckets;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.compaction.CompactionRecord;
 
 public class AggregatedNamespaceStats {
@@ -65,6 +66,8 @@ public class AggregatedNamespaceStats {
     StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC);
     int delayedMessageIndexSizeInBytes;
 
+    Map bucketDelayedIndexStats = new HashMap<>();
+
     void updateStats(TopicStats stats) {
         topicsCount++;
 
@@ -83,6 +86,14 @@ void updateStats(TopicStats stats) {
         msgOutCounter += stats.msgOutCounter;
         delayedMessageIndexSizeInBytes += stats.delayedMessageIndexSizeInBytes;
 
+        stats.bucketDelayedIndexStats.forEach((k, v) -> {
+            TopicMetricBean topicMetricBean =
+                    bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+            topicMetricBean.name = v.name;
+            topicMetricBean.labelsAndValues = v.labelsAndValues;
+            topicMetricBean.value += v.value;
+        });
+
         this.ongoingTxnCount += stats.ongoingTxnCount;
         this.abortedTxnCount += stats.abortedTxnCount;
         this.committedTxnCount += stats.committedTxnCount;
@@ -132,6 +143,13 @@ void updateStats(TopicStats stats) {
             subsStats.filterRejectedMsgCount += as.filterRejectedMsgCount;
             subsStats.filterRescheduledMsgCount += as.filterRescheduledMsgCount;
             subsStats.delayedMessageIndexSizeInBytes += as.delayedMessageIndexSizeInBytes;
+            as.bucketDelayedIndexStats.forEach((k, v) -> {
+                TopicMetricBean topicMetricBean =
+                        subsStats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+                topicMetricBean.name = v.name;
+                topicMetricBean.labelsAndValues = v.labelsAndValues;
+                topicMetricBean.value += v.value;
+            });
             as.consumerStat.forEach((c, v) -> {
                 AggregatedConsumerStats consumerStats =
                         subsStats.consumerStat.computeIfAbsent(c, k -> new AggregatedConsumerStats());
@@ -172,5 +190,6 @@ public void reset() {
         replicationStats.clear();
         subscriptionStats.clear();
         delayedMessageIndexSizeInBytes = 0;
+        bucketDelayedIndexStats.clear();
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java
index 383c671754dc1..da0324c55655c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 
 public class AggregatedSubscriptionStats {
 
@@ -75,4 +76,6 @@ public class AggregatedSubscriptionStats {
     public Map consumerStat = new HashMap<>();
 
     long delayedMessageIndexSizeInBytes;
+
+    public Map bucketDelayedIndexStats = new HashMap<>();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
index 918aef539cff4..32fb06ea3ce8c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
@@ -28,6 +28,7 @@
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
@@ -155,6 +156,7 @@ private static void aggregateTopicStats(TopicStats stats, SubscriptionStatsImpl
         subsStats.filterRejectedMsgCount = subscriptionStats.filterRejectedMsgCount;
         subsStats.filterRescheduledMsgCount = subscriptionStats.filterRescheduledMsgCount;
         subsStats.delayedMessageIndexSizeInBytes = subscriptionStats.delayedMessageIndexSizeInBytes;
+        subsStats.bucketDelayedIndexStats = subscriptionStats.bucketDelayedIndexStats;
     }
 
     private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics,
@@ -197,6 +199,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include
         stats.averageMsgSize = tStatus.averageMsgSize;
         stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes;
         stats.delayedMessageIndexSizeInBytes = tStatus.delayedMessageIndexSizeInBytes;
+        stats.bucketDelayedIndexStats = tStatus.bucketDelayedIndexStats;
         stats.abortedTxnCount = tStatus.abortedTxnCount;
         stats.ongoingTxnCount = tStatus.ongoingTxnCount;
         stats.committedTxnCount = tStatus.committedTxnCount;
@@ -379,6 +382,10 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat
         writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes, cluster,
                 namespace);
 
+        stats.bucketDelayedIndexStats.forEach((k, metric) -> {
+            writeMetric(stream, metric.name, metric.value, cluster, namespace, metric.labelsAndValues);
+        });
+
         writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace);
 
         stats.managedLedgerStats.storageWriteLatencyBuckets.refresh();
@@ -472,8 +479,10 @@ private static void writeMetric(PrometheusMetricStreams stream, String metricNam
     }
 
     private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster,
-                                    String namespace) {
-        stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace);
+                                    String namespace, String... extraLabelsAndValues) {
+        String[] labelsAndValues = new String[]{"cluster", cluster, "namespace", namespace};
+        String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues);
+        stream.writeSample(metricName, value, labels);
     }
 
     private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
index abc6979484e58..3a2563a87587b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
@@ -25,6 +25,7 @@
 import org.apache.bookkeeper.mledger.util.StatsBuckets;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.compaction.CompactionRecord;
 import org.apache.pulsar.compaction.CompactorMXBean;
 
@@ -70,6 +71,8 @@ class TopicStats {
     StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC);
     public long delayedMessageIndexSizeInBytes;
 
+    Map bucketDelayedIndexStats = new HashMap<>();
+
     public void reset() {
         subscriptionsCount = 0;
         producersCount = 0;
@@ -107,6 +110,7 @@ public void reset() {
         compactionCompactedEntriesSize = 0;
         compactionLatencyBuckets.reset();
         delayedMessageIndexSizeInBytes = 0;
+        bucketDelayedIndexStats.clear();
     }
 
     public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats,
@@ -162,6 +166,11 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
         writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes,
                 cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
 
+        for (TopicMetricBean topicMetricBean : stats.bucketDelayedIndexStats.values()) {
+            writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace,
+                    topic, splitTopicAndPartitionIndexLabel, topicMetricBean.labelsAndValues);
+        }
+
         long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets();
         writeMetric(stream, "pulsar_storage_write_latency_le_0_5",
                 latencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
@@ -310,6 +319,13 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
                     subsStats.delayedMessageIndexSizeInBytes, cluster, namespace, topic, sub,
                     splitTopicAndPartitionIndexLabel);
 
+            final String[] subscriptionLabel = {"subscription", sub};
+            for (TopicMetricBean topicMetricBean : subsStats.bucketDelayedIndexStats.values()) {
+                String[] labelsAndValues = ArrayUtils.addAll(subscriptionLabel, topicMetricBean.labelsAndValues);
+                writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace,
+                        topic, splitTopicAndPartitionIndexLabel, labelsAndValues);
+            }
+
             subsStats.consumerStat.forEach((c, consumerStats) -> {
                 writeConsumerMetric(stream, "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver,
                         cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel);
@@ -409,6 +425,12 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
             writeMetric(stream, "pulsar_compaction_latency_count",
                     stats.compactionLatencyBuckets.getCount(), cluster, namespace, topic,
                     splitTopicAndPartitionIndexLabel);
+
+            for (TopicMetricBean topicMetricBean : stats.bucketDelayedIndexStats.values()) {
+                String[] labelsAndValues = topicMetricBean.labelsAndValues;
+                writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace,
+                        topic, splitTopicAndPartitionIndexLabel, labelsAndValues);
+            }
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
index cf7310c7067c3..9e924bdeda341 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
@@ -174,7 +174,7 @@ public CompletableFuture getBucketSnapshotLength(long bucketId) {
             long length = 0;
             List bufList = this.bucketSnapshots.get(bucketId);
             for (ByteBuf byteBuf : bufList) {
-                length += byteBuf.array().length;
+                length += byteBuf.readableBytes();
             }
             return length;
         }, executorService);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 717bada7705dc..95234d688f6a0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -46,7 +46,6 @@
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest;
-import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage;
 import org.apache.pulsar.broker.delayed.MockManagedCursor;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
@@ -158,7 +157,7 @@ public Object[][] provider(Method method) throws Exception {
     }
 
     @Test(dataProvider = "delayedTracker")
-    public void testContainsMessage(DelayedDeliveryTracker tracker) {
+    public void testContainsMessage(BucketDelayedDeliveryTracker tracker) {
         tracker.addMessage(1, 1, 10);
         tracker.addMessage(2, 2, 20);
 
@@ -191,6 +190,12 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
 
         clockTime.set(1 * 10);
 
+        Awaitility.await().untilAsserted(() -> {
+            Assert.assertTrue(
+                    tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging ||
+                            !x.getSnapshotCreateFuture().get().isDone()));
+        });
+
         assertTrue(tracker.hasMessageAvailable());
         Set scheduledMessages = tracker.getScheduledMessages(100);
 
@@ -202,16 +207,16 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
 
         clockTime.set(30 * 10);
 
-        tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,50);
+        BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50);
 
-        assertFalse(tracker.containsMessage(101, 101));
-        assertEquals(tracker.getNumberOfDelayedMessages(), 70);
+        assertFalse(tracker2.containsMessage(101, 101));
+        assertEquals(tracker2.getNumberOfDelayedMessages(), 70);
 
         clockTime.set(100 * 10);
 
-        assertTrue(tracker.hasMessageAvailable());
-        scheduledMessages = tracker.getScheduledMessages(70);
+        assertTrue(tracker2.hasMessageAvailable());
+        scheduledMessages = tracker2.getScheduledMessages(70);
 
         assertEquals(scheduledMessages.size(), 70);
 
@@ -221,7 +226,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
             i++;
         }
 
-        tracker.close();
+        tracker2.close();
     }
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 09b7cbbf1b99d..5480a2e7a704a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -19,18 +19,26 @@
 package org.apache.pulsar.broker.service.persistent;
 
 import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
+import com.google.common.collect.Multimap;
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
 import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.service.Dispatcher;
+import org.apache.pulsar.broker.stats.PrometheusMetricsTest;
+import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.Schema;
@@ -162,4 +170,139 @@ public void testUnsubscribe() throws Exception {
             }
         }
     }
+
+
+    @Test
+    public void testBucketDelayedIndexMetrics() throws Exception {
+        cleanup();
+        setup();
+
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testBucketDelayedIndexMetrics");
+
+        @Cleanup
+        Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("test_sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("test_sub2")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        final int N = 101;
+
+        for (int i = 0; i < N; i++) {
+            producer.newMessage()
+                    .value("msg-" + i)
+                    .deliverAfter(3600 + i, TimeUnit.SECONDS)
+                    .sendAsync();
+        }
+        producer.flush();
+
+        Thread.sleep(2000);
+
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, true, true, true, output);
+        String metricsStr = output.toString(StandardCharsets.UTF_8);
+        Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr);
+
+        List bucketsMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_total").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt bucketsSum = new MutableInt();
+        bucketsMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")).forEach(metric -> {
+            assertEquals(3, metric.value);
+            bucketsSum.add(metric.value);
+        });
+        assertEquals(6, bucketsSum.intValue());
+        Optional bucketsTopicMetric =
+                bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(bucketsTopicMetric.isPresent());
+        assertEquals(bucketsSum.intValue(), bucketsTopicMetric.get().value);
+
+        List loadedIndexMetrics =
+                metricsMap.get("pulsar_delayed_message_index_loaded").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt loadedIndexSum = new MutableInt();
+        long count = loadedIndexMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")).peek(metric -> {
+            assertTrue(metric.value > 0 && metric.value <= N);
+            loadedIndexSum.add(metric.value);
+        }).count();
+        assertEquals(2, count);
+        Optional loadedIndexTopicMetrics =
+                bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(loadedIndexTopicMetrics.isPresent());
+        assertEquals(loadedIndexSum.intValue(), loadedIndexTopicMetrics.get().value);
+
+        List snapshotSizeBytesMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_snapshot_size_bytes").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt snapshotSizeBytesSum = new MutableInt();
+        count = snapshotSizeBytesMetrics.stream().filter(metric -> metric.tags.containsKey("subscription"))
+                .peek(metric -> {
+                    assertTrue(metric.value > 0);
+                    snapshotSizeBytesSum.add(metric.value);
+                }).count();
+        assertEquals(2, count);
+        Optional snapshotSizeBytesTopicMetrics =
+                snapshotSizeBytesMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(snapshotSizeBytesTopicMetrics.isPresent());
+        assertEquals(snapshotSizeBytesSum.intValue(), snapshotSizeBytesTopicMetrics.get().value);
+
+        List opCountMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_op_count").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt opCountMetricsSum = new MutableInt();
+        count = opCountMetrics.stream()
+                .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type").equals("create")
+                        && metric.tags.containsKey("subscription"))
+                .peek(metric -> {
+                    assertTrue(metric.value >= 2);
+                    opCountMetricsSum.add(metric.value);
+                }).count();
+        assertEquals(2, count);
+        Optional opCountTopicMetrics =
+                opCountMetrics.stream()
+                        .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type")
+                                .equals("create") && !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(opCountTopicMetrics.isPresent());
+        assertEquals(opCountMetricsSum.intValue(), opCountTopicMetrics.get().value);
+
+        List opLatencyMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_op_latency_ms").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt opLatencyMetricsSum = new MutableInt();
+        count = opLatencyMetrics.stream()
+                .filter(metric -> metric.tags.get("type").equals("create")
+                        && metric.tags.containsKey("subscription"))
+                .peek(metric -> {
+                    assertTrue(metric.tags.containsKey("quantile"));
+                    opLatencyMetricsSum.add(metric.value);
+                }).count();
+        assertTrue(count >= 2);
+        Optional opLatencyTopicMetrics =
+                opCountMetrics.stream()
+                        .filter(metric -> metric.tags.get("type").equals("create")
+                                && !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(opLatencyTopicMetrics.isPresent());
+        assertEquals(opLatencyMetricsSum.intValue(), opLatencyTopicMetrics.get().value);
+
+        ByteArrayOutputStream namespaceOutput = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, false, true, true, namespaceOutput);
+        Multimap namespaceMetricsMap = PrometheusMetricsTest.parseMetrics(namespaceOutput.toString(StandardCharsets.UTF_8));
+
+        Optional namespaceMetric =
+                namespaceMetricsMap.get("pulsar_delayed_message_index_bucket_total").stream().findFirst();
+        assertTrue(namespaceMetric.isPresent());
+        assertEquals(6, namespaceMetric.get().value);
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index 9c7e24ba02118..25fa666523f3c 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -134,6 +134,8 @@ public class SubscriptionStatsImpl implements SubscriptionStats {
     /** The size of InMemoryDelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
+    public Map bucketDelayedIndexStats;
+
     /** SubscriptionProperties (key/value strings) associated with this subscribe. */
     public Map subscriptionProperties;
 
@@ -149,6 +151,7 @@ public SubscriptionStatsImpl() {
         this.consumers = new ArrayList<>();
         this.consumersAfterMarkDeletePosition = new LinkedHashMap<>();
         this.subscriptionProperties = new HashMap<>();
+        this.bucketDelayedIndexStats = new HashMap<>();
     }
 
     public void reset() {
@@ -175,6 +178,7 @@ public void reset() {
         filterAcceptedMsgCount = 0;
         filterRejectedMsgCount = 0;
         filterRescheduledMsgCount = 0;
+        bucketDelayedIndexStats.clear();
     }
 
     // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current
@@ -215,6 +219,14 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) {
         this.filterAcceptedMsgCount += stats.filterAcceptedMsgCount;
         this.filterRejectedMsgCount += stats.filterRejectedMsgCount;
         this.filterRescheduledMsgCount += stats.filterRescheduledMsgCount;
+        stats.bucketDelayedIndexStats.forEach((k, v) -> {
+            TopicMetricBean topicMetricBean =
+                    this.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+            topicMetricBean.name = v.name;
+            topicMetricBean.labelsAndValues = v.labelsAndValues;
+            topicMetricBean.value += v.value;
+        });
+
         return this;
     }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java
new file mode 100644
index 0000000000000..e01a9d7aa71f3
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.policies.data.stats;
+
+import lombok.AllArgsConstructor;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor
+@AllArgsConstructor
+public class TopicMetricBean {
+    public String name;
+    public double value;
+    public String[] labelsAndValues;
+}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index 12d30124f7dcd..c9c4739b904f6 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -139,6 +139,9 @@ public class TopicStatsImpl implements TopicStats {
     /** The size of InMemoryDelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
+    /** Map of bucket delayed index statistics. */
+    public Map bucketDelayedIndexStats;
+
     /** The compaction stats. */
     public CompactionStatsImpl compaction;
 
@@ -182,6 +185,7 @@ public TopicStatsImpl() {
         this.subscriptions = new HashMap<>();
         this.replication = new TreeMap<>();
         this.compaction = new CompactionStatsImpl();
+        this.bucketDelayedIndexStats = new HashMap<>();
     }
 
     public void reset() {
@@ -214,6 +218,7 @@ public void reset() {
         this.delayedMessageIndexSizeInBytes = 0;
         this.compaction.reset();
         this.ownerBroker = null;
+        this.bucketDelayedIndexStats.clear();
     }
 
     // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current
@@ -244,6 +249,14 @@ public TopicStatsImpl add(TopicStats ts) {
         this.abortedTxnCount = stats.abortedTxnCount;
         this.committedTxnCount = stats.committedTxnCount;
 
+        stats.bucketDelayedIndexStats.forEach((k, v) -> {
+            TopicMetricBean topicMetricBean =
+                    this.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+            topicMetricBean.name = v.name;
+            topicMetricBean.labelsAndValues = v.labelsAndValues;
+            topicMetricBean.value += v.value;
+        });
+
         for (int index = 0; index < stats.getPublishers().size(); index++) {
            PublisherStats s = stats.getPublishers().get(index);
            if (s.isSupportsPartialProducer() && s.getProducerName() != null) {

From 02147454c425b92f0cd1caefa73b9339db6a0269 Mon Sep 17 00:00:00 2001
From: tison 
Date: Sat, 1 Apr 2023 14:18:02 +0800
Subject: [PATCH 128/404] [feat][client] Support configure MessageCrypto in
 ProducerBuilder (#19939)

Signed-off-by: tison 
---
 .../pulsar/client/api/ProducerBuilder.java    | 11 ++++++
 .../client/impl/ProducerBuilderImpl.java      |  7 ++++
 .../client/impl/ProducerBuilderImplTest.java  | 39 +++++++++++--------
 3 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java
index d2231e71c237a..896c22313247a 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java
@@ -389,6 +389,17 @@ public interface ProducerBuilder extends Cloneable {
      */
     ProducerBuilder defaultCryptoKeyReader(Map publicKeys);
 
+    /**
+     * Sets a {@link MessageCrypto}.
+     *
+     * 
Contains methods to encrypt/decrypt messages for end-to-end encryption.
+     *
+     * @param messageCrypto
+     *            MessageCrypto object
+     * @return the producer builder instance
+     */
+    ProducerBuilder messageCrypto(MessageCrypto messageCrypto);
+
     /**
      * Add public encryption key, used by producer to encrypt the data key.
      *
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java
index bfdbb63de1855..ecbdfa76b64ae 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.CryptoKeyReader;
 import org.apache.pulsar.client.api.HashingScheme;
+import org.apache.pulsar.client.api.MessageCrypto;
 import org.apache.pulsar.client.api.MessageRouter;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
@@ -221,6 +222,12 @@ public ProducerBuilder defaultCryptoKeyReader(@NonNull Map pu
         return cryptoKeyReader(DefaultCryptoKeyReader.builder().publicKeys(publicKeys).build());
     }
 
+    @Override
+    public ProducerBuilder messageCrypto(MessageCrypto messageCrypto) {
+        conf.setMessageCrypto(messageCrypto);
+        return this;
+    }
+
     @Override
     public ProducerBuilder addEncryptionKey(String key) {
         checkArgument(StringUtils.isNotBlank(key), "Encryption key cannot be blank");
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java
index 66c552ef7934a..bb3e3fc3accf6 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java
@@ -18,6 +18,15 @@
  */
 package org.apache.pulsar.client.impl;
 
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertNotNull;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageRouter;
 import org.apache.pulsar.client.api.MessageRoutingMode;
@@ -26,20 +35,10 @@
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TopicMetadata;
 import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
+import org.apache.pulsar.client.impl.crypto.MessageCryptoBc;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertNotNull;
-
 /**
  * Unit tests of {@link ProducerBuilderImpl}.
  */
@@ -47,13 +46,13 @@ public class ProducerBuilderImplTest {
 
     private static final String TOPIC_NAME = "testTopicName";
     private PulsarClientImpl client;
-    private ProducerBuilderImpl producerBuilderImpl;
+    private ProducerBuilderImpl producerBuilderImpl;
 
     @BeforeClass(alwaysRun = true)
     public void setup() {
-        Producer producer = mock(Producer.class);
+        Producer producer = mock(Producer.class);
         client = mock(PulsarClientImpl.class);
-        producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES);
+        producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES);
         when(client.newProducer()).thenReturn(producerBuilderImpl);
 
         when(client.createProducerAsync(
@@ -66,8 +65,8 @@ public void testProducerBuilderImpl() throws PulsarClientException {
         Map properties = new HashMap<>();
         properties.put("Test-Key2", "Test-Value2");
 
-        producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES);
-        Producer producer = producerBuilderImpl.topic(TOPIC_NAME)
+        producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES);
+        Producer producer = producerBuilderImpl.topic(TOPIC_NAME)
                 .producerName("Test-Producer")
                 .maxPendingMessages(2)
                 .addEncryptionKey("Test-EncryptionKey")
@@ -78,6 +77,14 @@ public void testProducerBuilderImpl() throws PulsarClientException {
         assertNotNull(producer);
     }
 
+    @Test
+    public void testProducerBuilderImplWhenMessageCryptoSet() throws PulsarClientException {
+        producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES);
+        producerBuilderImpl.topic(TOPIC_NAME).messageCrypto(new MessageCryptoBc("ctx1", true));
+        assertNotNull(producerBuilderImpl.create());
+        assertNotNull(producerBuilderImpl.getConf().getMessageCrypto());
+    }
+
     @Test
     public void testProducerBuilderImplWhenMessageRoutingModeAndMessageRouterAreNotSet() throws PulsarClientException {
         producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES);

From be57e9a79b0b6fd40ef02414df7238050b7a885d Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Sat, 1 Apr 2023 19:21:58 +0800
Subject: [PATCH 129/404] [fix][schema]Fix AutoProduceBytes producer can not be
 created to a topic with ProtoBuf schema (#19767)

### Motivation
1. There is a topic1 with a protobuf schema.
2. Create a producer1 with AutoProduceBytes schema.
3. The producer1 will be created failed because the way to get the schema of protobuf schema is not supported. ### ###
### Modification
Because the Protobuf schema is implemented from the AvroBaseStructSchema. So we add a way to get Protobuf schema just like the AvroSchema.
---
 .../org/apache/pulsar/schema/SchemaTest.java    | 17 +++++++++++++++++
 .../pulsar/client/impl/PulsarClientImpl.java    |  9 ++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
index d99496f4a967b..0ec72b2ef470a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.client.api.schema.GenericRecord;
 import org.apache.pulsar.client.api.schema.SchemaDefinition;
 import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl;
+import org.apache.pulsar.client.impl.schema.ProtobufSchema;
 import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
 import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord;
 import org.apache.pulsar.client.impl.schema.writer.AvroWriter;
@@ -113,6 +114,22 @@ public void cleanup() throws Exception {
         super.internalCleanup();
     }
 
+    @Test
+    public void testGetSchemaWhenCreateAutoProduceBytesProducer() throws Exception{
+        final String tenant = PUBLIC_TENANT;
+        final String namespace = "test-namespace-" + randomName(16);
+        final String topic = tenant + "/" + namespace + "/test-getSchema";
+        admin.namespaces().createNamespace(
+                tenant + "/" + namespace,
+                Sets.newHashSet(CLUSTER_NAME)
+        );
+
+        ProtobufSchema protobufSchema =
+                ProtobufSchema.of(org.apache.pulsar.client.api.schema.proto.Test.TestMessage.class);
+        pulsarClient.newProducer(protobufSchema).topic(topic).create();
+        pulsarClient.newProducer(org.apache.pulsar.client.api.Schema.AUTO_PRODUCE_BYTES()).topic(topic).create();
+    }
+
     @Test
     public void testMultiTopicSetSchemaProvider() throws Exception {
         final String tenant = PUBLIC_TENANT;
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
index ebc11f44ce749..f37709f3d84f6 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.client.impl.conf.ReaderConfigurationData;
 import org.apache.pulsar.client.impl.schema.AutoConsumeSchema;
 import org.apache.pulsar.client.impl.schema.AutoProduceBytesSchema;
+import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
 import org.apache.pulsar.client.impl.schema.generic.MultiVersionSchemaInfoProvider;
 import org.apache.pulsar.client.impl.transaction.TransactionBuilderImpl;
 import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl;
@@ -81,6 +82,7 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.schema.SchemaInfo;
+import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
@@ -350,7 +352,12 @@ public  CompletableFuture> createProducerAsync(ProducerConfigurat
             return lookup.getSchema(TopicName.get(conf.getTopicName()))
                     .thenCompose(schemaInfoOptional -> {
                         if (schemaInfoOptional.isPresent()) {
-                            autoProduceBytesSchema.setSchema(Schema.getSchema(schemaInfoOptional.get()));
+                            SchemaInfo schemaInfo = schemaInfoOptional.get();
+                            if (schemaInfo.getType() == SchemaType.PROTOBUF) {
+                                autoProduceBytesSchema.setSchema(new GenericAvroSchema(schemaInfo));
+                            } else {
+                                autoProduceBytesSchema.setSchema(Schema.getSchema(schemaInfo));
+                            }
                         } else {
                             autoProduceBytesSchema.setSchema(Schema.BYTES);
                         }

From 2dcaf0ef692c929ebf6c510d328e0f2775f07f3e Mon Sep 17 00:00:00 2001
From: LinChen 
Date: Mon, 3 Apr 2023 12:55:16 +0800
Subject: [PATCH 130/404] [fix][broker] Fix the loss of bundle stats data
 reported to zookeeper, when the updateStats method is executed (#19887)

Co-authored-by: lordcheng10 
---
 .../java/org/apache/pulsar/broker/service/PulsarStats.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
index 9cdf9d1dfc68d..2059aa04350d4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
@@ -233,7 +233,7 @@ public synchronized void updateStats(
         updatedAt = System.currentTimeMillis();
     }
 
-    public NamespaceBundleStats invalidBundleStats(String bundleName) {
+    public synchronized NamespaceBundleStats invalidBundleStats(String bundleName) {
         return bundleStats.remove(bundleName);
     }
 
@@ -254,7 +254,7 @@ public BrokerOperabilityMetrics getBrokerOperabilityMetrics() {
         return brokerOperabilityMetrics;
     }
 
-    public Map getBundleStats() {
+    public synchronized Map getBundleStats() {
         return bundleStats;
     }
 

From 67eb0fb4e830f2fb7affc9b3ab77b5423b4ec4ce Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 3 Apr 2023 01:31:35 -0500
Subject: [PATCH 131/404] [cleanup][proxy] ProxyConnection should not call
 super.exceptionCaught (#19990)

### Motivation

While debugging an issue, I noticed that we call `super.exceptionCaught(ctx, cause);` in the `ProxyConnection` class. This leads to the following log line:

> An exceptionCaught() event was fired, and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception. io.netty.channel.unix.Errors$NativeIoException: recvAddress(..) failed: Connection reset by peer

Because we always handle exceptions, there is no need to forward them to the next handler.

### Modifications

* Remove a single method call

### Verifying this change

This is a trivial change. Note that we do not call the super method in any other handler implementations in the project.

### Documentation

- [x] `doc-not-needed` 

### Matching PR in forked repository

PR in forked repository: skipping PR for this trivial change
---
 .../java/org/apache/pulsar/proxy/server/ProxyConnection.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 1e19d760c6d7b..b7f5534ee81fb 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -208,7 +208,6 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-        super.exceptionCaught(ctx, cause);
         LOG.warn("[{}] Got exception {} : Message: {} State: {}", remoteAddress, cause.getClass().getSimpleName(),
                 cause.getMessage(), state,
                 ClientCnx.isKnownException(cause) ? null : cause);

From 5ef3a21a068b837a56a432361aa4b33732a13cfb Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 3 Apr 2023 15:59:17 +0800
Subject: [PATCH 132/404] [fix][build] Client modules should be built with Java
 8 (#19991)

---
 pom.xml                           |  4 +--
 pulsar-client-auth-athenz/pom.xml | 41 ++++++++++++++++++++++++++++++-
 pulsar-client-auth-sasl/pom.xml   | 39 +++++++++++++++++++++++++++++
 3 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4c489355281ba..9f6f2909c1af3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1966,8 +1966,8 @@ flexible messaging model and an intuitive client API.
         
         8
         8
-        
-        
+        
+        
       
     
     
diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml
index 0b757d747872a..3cdbdb48a2935 100644
--- a/pulsar-client-auth-athenz/pom.xml
+++ b/pulsar-client-auth-athenz/pom.xml
@@ -61,10 +61,49 @@
       org.apache.commons
       commons-lang3
     
-
   
+
   
     
+      
+        org.apache.maven.plugins
+        maven-compiler-plugin
+        
+          ${pulsar.client.compiler.release}
+        
+      
+
+      
+        org.apache.maven.plugins
+        maven-enforcer-plugin
+        ${maven-enforcer-plugin.version}
+        
+          
+            enforce-bytecode-version
+            
+              enforce
+            
+            
+              
+                
+                  ${pulsar.client.compiler.release}
+                  
+                    test
+                  
+                
+              
+            
+          
+        
+        
+          
+            org.codehaus.mojo
+            extra-enforcer-rules
+            ${extra-enforcer-rules.version}
+          
+        
+      
+
       
         org.gaul
         modernizer-maven-plugin
diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml
index 9744a230cf3af..c1143bf69331e 100644
--- a/pulsar-client-auth-sasl/pom.xml
+++ b/pulsar-client-auth-sasl/pom.xml
@@ -71,6 +71,45 @@
 
   
     
+      
+        org.apache.maven.plugins
+        maven-compiler-plugin
+        
+          ${pulsar.client.compiler.release}
+        
+      
+
+      
+        org.apache.maven.plugins
+        maven-enforcer-plugin
+        ${maven-enforcer-plugin.version}
+        
+          
+            enforce-bytecode-version
+            
+              enforce
+            
+            
+              
+                
+                  ${pulsar.client.compiler.release}
+                  
+                    test
+                  
+                
+              
+            
+          
+        
+        
+          
+            org.codehaus.mojo
+            extra-enforcer-rules
+            ${extra-enforcer-rules.version}
+          
+        
+      
+
       
         org.gaul
         modernizer-maven-plugin

From d1fc7323cbf61a6d2955486fc123fdde5253e72c Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Tue, 4 Apr 2023 10:31:33 +0800
Subject: [PATCH 133/404] [fix][broker] Ignore and remove the replicator cursor
 when the remote cluster is absent (#19972)

---
 .../service/persistent/PersistentTopic.java   | 30 +++++++--
 .../persistent/PersistentTopicTest.java       | 63 +++++++++++++++++++
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index fa08330ff3c35..18a662c4b7a38 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -1696,14 +1696,32 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
         return future;
     }
 
+    private CompletableFuture checkReplicationCluster(String remoteCluster) {
+        return brokerService.getPulsar().getPulsarResources().getNamespaceResources()
+                .getPoliciesAsync(TopicName.get(topic).getNamespaceObject())
+                .thenApply(optPolicies -> optPolicies.map(policies -> policies.replication_clusters)
+                        .orElse(Collections.emptySet()).contains(remoteCluster)
+                        || topicPolicies.getReplicationClusters().get().contains(remoteCluster));
+    }
+
     protected CompletableFuture addReplicationCluster(String remoteCluster, ManagedCursor cursor,
             String localCluster) {
         return AbstractReplicator.validatePartitionedTopicAsync(PersistentTopic.this.getName(), brokerService)
-                .thenCompose(__ -> brokerService.pulsar().getPulsarResources().getClusterResources()
-                        .getClusterAsync(remoteCluster)
-                        .thenApply(clusterData ->
-                                brokerService.getReplicationClient(remoteCluster, clusterData)))
+                .thenCompose(__ -> checkReplicationCluster(remoteCluster))
+                .thenCompose(clusterExists -> {
+                    if (!clusterExists) {
+                        log.warn("Remove the replicator because the cluster '{}' does not exist", remoteCluster);
+                        return removeReplicator(remoteCluster).thenApply(__ -> null);
+                    }
+                    return brokerService.pulsar().getPulsarResources().getClusterResources()
+                            .getClusterAsync(remoteCluster)
+                            .thenApply(clusterData ->
+                                    brokerService.getReplicationClient(remoteCluster, clusterData));
+                })
                 .thenAccept(replicationClient -> {
+                    if (replicationClient == null) {
+                        return;
+                    }
                     Replicator replicator = replicators.computeIfAbsent(remoteCluster, r -> {
                         try {
                             return new GeoPersistentReplicator(PersistentTopic.this, cursor, localCluster,
@@ -1727,8 +1745,8 @@ CompletableFuture removeReplicator(String remoteCluster) {
 
         String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster);
 
-        replicators.get(remoteCluster).disconnect().thenRun(() -> {
-
+        Optional.ofNullable(replicators.get(remoteCluster)).map(Replicator::disconnect)
+                .orElse(CompletableFuture.completedFuture(null)).thenRun(() -> {
             ledger.asyncDeleteCursor(name, new DeleteCursorCallback() {
                 @Override
                 public void deleteCursorComplete(Object ctx) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
index 80a79e0234de4..c63be7aad01cd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
@@ -40,15 +40,20 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 import lombok.Cleanup;
 import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.BrokerTestBase;
@@ -57,6 +62,7 @@
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
@@ -66,7 +72,9 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.Policies;
+import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.policies.data.TopicStats;
 import org.awaitility.Awaitility;
 import org.junit.Assert;
@@ -525,4 +533,59 @@ public void testDeleteTopicFail() throws Exception {
         makeDeletedFailed.set(false);
         persistentTopic.delete().get();
     }
+
+    @DataProvider(name = "topicLevelPolicy")
+    public static Object[][] topicLevelPolicy() {
+        return new Object[][] { { true }, { false } };
+    }
+
+    @Test(dataProvider = "topicLevelPolicy")
+    public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) throws Exception {
+        final String namespace = "prop/ns-abc";
+        final String topicName = "persistent://" + namespace
+                + "/testCreateTopicWithZombieReplicatorCursor" + topicLevelPolicy;
+        final String remoteCluster = "remote";
+        admin.topics().createNonPartitionedTopic(topicName);
+        admin.topics().createSubscription(topicName, conf.getReplicatorPrefix() + "." + remoteCluster,
+                MessageId.earliest, true);
+
+        admin.clusters().createCluster(remoteCluster, ClusterData.builder()
+                .serviceUrl("http://localhost:11112")
+                .brokerServiceUrl("pulsar://localhost:11111")
+                .build());
+        TenantInfo tenantInfo = admin.tenants().getTenantInfo("prop");
+        tenantInfo.getAllowedClusters().add(remoteCluster);
+        admin.tenants().updateTenant("prop", tenantInfo);
+
+        if (topicLevelPolicy) {
+            admin.topics().setReplicationClusters(topicName, Collections.singletonList(remoteCluster));
+        } else {
+            admin.namespaces().setNamespaceReplicationClustersAsync(
+                    namespace, Collections.singleton(remoteCluster)).get();
+        }
+
+        final PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false)
+                .get(3, TimeUnit.SECONDS).orElse(null);
+        assertNotNull(topic);
+
+        final Supplier> getCursors = () -> {
+            final Set cursors = new HashSet<>();
+            final Iterable iterable = topic.getManagedLedger().getCursors();
+            iterable.forEach(c -> cursors.add(c.getName()));
+            return cursors;
+        };
+        assertEquals(getCursors.get(), Collections.singleton(conf.getReplicatorPrefix() + "." + remoteCluster));
+
+        if (topicLevelPolicy) {
+            admin.topics().setReplicationClusters(topicName, Collections.emptyList());
+        } else {
+            admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.emptySet()).get();
+        }
+        admin.clusters().deleteCluster(remoteCluster);
+        // Now the cluster and its related policy has been removed but the replicator cursor still exists
+
+        topic.initialize().get(3, TimeUnit.SECONDS);
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .until(() -> !topic.getManagedLedger().getCursors().iterator().hasNext());
+    }
 }

From 8c50a6c2e91c81dbf187ce5e66cb39e2758a741e Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Tue, 4 Apr 2023 15:52:34 +0800
Subject: [PATCH 134/404] [improve][client] PIP-229: Add a common interface to
 get fields of MessageIdData (#19414)

---
 .../service/PersistentFailoverE2ETest.java    |   8 +-
 .../broker/service/SubscriptionSeekTest.java  |   2 +-
 .../client/api/CustomMessageIdTest.java       | 142 +++++++++++
 .../api/PartitionedProducerConsumerTest.java  |   2 +-
 .../pulsar/client/impl/ConsumerAckTest.java   |   4 +-
 .../pulsar/client/impl/MessageIdTest.java     |   3 -
 .../pulsar/client/api/MessageIdAdv.java       | 122 ++++++++++
 .../pulsar/client/api/TopicMessageId.java     |  43 +++-
 .../impl/AcknowledgmentsGroupingTracker.java  |   2 +-
 .../pulsar/client/impl/BatchMessageAcker.java |  95 --------
 .../impl/BatchMessageAckerDisabled.java       |  50 ----
 .../client/impl/BatchMessageIdImpl.java       |  82 +++----
 .../client/impl/ChunkMessageIdImpl.java       |   9 +-
 .../pulsar/client/impl/ConsumerBase.java      |  15 +-
 .../pulsar/client/impl/ConsumerImpl.java      | 222 +++++++-----------
 .../pulsar/client/impl/MessageIdAdvUtils.java |  74 ++++++
 .../pulsar/client/impl/MessageIdImpl.java     |  71 +-----
 .../pulsar/client/impl/MessageImpl.java       |   8 +-
 .../impl/MessagePayloadContextImpl.java       |   9 +-
 .../client/impl/MultiTopicsConsumerImpl.java  |  44 ++--
 .../client/impl/NegativeAcksTracker.java      |  10 +-
 ...rsistentAcknowledgmentGroupingTracker.java |   2 +-
 ...sistentAcknowledgmentsGroupingTracker.java | 158 ++++++-------
 .../pulsar/client/impl/ResetCursorData.java   |  20 +-
 .../client/impl/TopicMessageIdImpl.java       |  39 +--
 .../pulsar/client/impl/TopicMessageImpl.java  |   2 +-
 .../client/impl/ZeroQueueConsumerImpl.java    |   3 +-
 .../src/main/resources/findbugsExclude.xml    |   5 +
 .../AcknowledgementsGroupingTrackerTest.java  |  13 +-
 .../impl/BatchMessageAckerDisabledTest.java   |  47 ----
 .../client/impl/BatchMessageAckerTest.java    |  83 -------
 .../client/impl/BatchMessageIdImplTest.java   |  33 +--
 .../impl/MessageIdSerializationTest.java      |   3 +-
 .../functions/utils/FunctionCommon.java       |   3 +-
 .../io/kafka/connect/KafkaConnectSink.java    |  36 ++-
 35 files changed, 686 insertions(+), 778 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java
 create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
 delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java
 delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java
 create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java
 delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java
 delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
index b263d4448d87a..f5895ec3761bf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
@@ -41,11 +41,11 @@
 import org.apache.pulsar.client.api.ConsumerEventListener;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -370,8 +370,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
             }
             totalMessages++;
             consumer1.acknowledge(msg);
-            MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(msg.getMessageId());
-            receivedPtns.add(msgId.getPartitionIndex());
+            receivedPtns.add(((MessageIdAdv) msg.getMessageId()).getPartitionIndex());
         }
 
         assertTrue(Sets.difference(listener1.activePtns, receivedPtns).isEmpty());
@@ -387,8 +386,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
             }
             totalMessages++;
             consumer2.acknowledge(msg);
-            MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(msg.getMessageId());
-            receivedPtns.add(msgId.getPartitionIndex());
+            receivedPtns.add(((MessageIdAdv) msg.getMessageId()).getPartitionIndex());
         }
         assertTrue(Sets.difference(listener1.inactivePtns, receivedPtns).isEmpty());
         assertTrue(Sets.difference(listener2.activePtns, receivedPtns).isEmpty());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
index 93f2a42bcda35..b11946069c9dd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
@@ -678,7 +678,7 @@ public void testSeekByFunction() throws Exception {
             if (message == null) {
                 break;
             }
-            received.add(MessageIdImpl.convertToMessageIdImpl(message.getMessageId()));
+            received.add(message.getMessageId());
         }
         int msgNumFromPartition1 = list.size() / 2;
         int msgNumFromPartition2 = 1;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java
new file mode 100644
index 0000000000000..52bfc9dda37e4
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-api")
+public class CustomMessageIdTest extends ProducerConsumerBase {
+
+    @BeforeClass
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @DataProvider
+    public static Object[][] enableBatching() {
+        return new Object[][]{
+                { true },
+                { false }
+        };
+    }
+
+    @Test
+    public void testSeek() throws Exception {
+        final var topic = "persistent://my-property/my-ns/test-seek-" + System.currentTimeMillis();
+        @Cleanup final var producer = pulsarClient.newProducer(Schema.INT32).topic(topic).create();
+        final var msgIds = new ArrayList();
+        for (int i = 0; i < 10; i++) {
+            msgIds.add(new SimpleMessageIdImpl((MessageIdAdv) producer.send(i)));
+        }
+        @Cleanup final var consumer = pulsarClient.newConsumer(Schema.INT32)
+                .topic(topic).subscriptionName("sub").subscribe();
+        consumer.seek(msgIds.get(6));
+        final var msg = consumer.receive(3, TimeUnit.SECONDS);
+        assertNotNull(msg);
+        assertEquals(msg.getValue(), 7);
+    }
+
+    @Test(dataProvider = "enableBatching")
+    public void testAcknowledgment(boolean enableBatching) throws Exception {
+        final var topic = "persistent://my-property/my-ns/test-ack-"
+                + enableBatching + System.currentTimeMillis();
+        final var producer = pulsarClient.newProducer(Schema.INT32)
+                .topic(topic)
+                .enableBatching(enableBatching)
+                .batchingMaxMessages(10)
+                .batchingMaxPublishDelay(300, TimeUnit.MILLISECONDS)
+                .create();
+        final var consumer = pulsarClient.newConsumer(Schema.INT32)
+                .topic(topic)
+                .subscriptionName("sub")
+                .enableBatchIndexAcknowledgment(true)
+                .isAckReceiptEnabled(true)
+                .subscribe();
+        for (int i = 0; i < 10; i++) {
+            producer.sendAsync(i);
+        }
+        final var msgIds = new ArrayList();
+        for (int i = 0; i < 10; i++) {
+            final var msg = consumer.receive();
+            final var msgId = new SimpleMessageIdImpl((MessageIdAdv) msg.getMessageId());
+            msgIds.add(msgId);
+            if (enableBatching) {
+                assertTrue(msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0);
+            } else {
+                assertFalse(msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0);
+            }
+        }
+        consumer.acknowledgeCumulative(msgIds.get(8));
+        consumer.redeliverUnacknowledgedMessages();
+        final var msg = consumer.receive(3, TimeUnit.SECONDS);
+        assertNotNull(msg);
+        assertEquals(msg.getValue(), 9);
+    }
+
+    private record SimpleMessageIdImpl(long ledgerId, long entryId, int batchIndex, int batchSize)
+            implements MessageIdAdv {
+
+        public SimpleMessageIdImpl(MessageIdAdv msgId) {
+            this(msgId.getLedgerId(), msgId.getEntryId(), msgId.getBatchIndex(), msgId.getBatchSize());
+        }
+
+        @Override
+        public byte[] toByteArray() {
+            return new byte[0]; // never used
+        }
+
+        @Override
+        public long getLedgerId() {
+            return ledgerId;
+        }
+
+        @Override
+        public long getEntryId() {
+            return entryId;
+        }
+
+        @Override
+        public int getBatchIndex() {
+            return batchIndex;
+        }
+
+        @Override
+        public int getBatchSize() {
+            return batchSize;
+        }
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
index cd384e587898d..13ae991a0e89c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
@@ -767,7 +767,7 @@ public void testMessageIdForSubscribeToSinglePartition() throws Exception {
 
             for (int i = 0; i < totalMessages; i ++) {
                 msg = consumer1.receive(5, TimeUnit.SECONDS);
-                Assert.assertEquals(MessageIdImpl.convertToMessageIdImpl(msg.getMessageId()).getPartitionIndex(), 2);
+                Assert.assertEquals(((MessageIdAdv) msg.getMessageId()).getPartitionIndex(), 2);
                 consumer1.acknowledge(msg);
             }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java
index 42da60906483c..a83283bc267b5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java
@@ -176,10 +176,10 @@ private AckTestData prepareDataForAck(String topic) throws PulsarClientException
             messageIds.add(message.getMessageId());
         }
         MessageId firstEntryMessageId = messageIds.get(0);
-        MessageId secondEntryMessageId = ((BatchMessageIdImpl) messageIds.get(1)).toMessageIdImpl();
+        MessageId secondEntryMessageId = MessageIdAdvUtils.discardBatch(messageIds.get(1));
         // Verify messages 2 to N must be in the same entry
         for (int i = 2; i < messageIds.size(); i++) {
-            assertEquals(((BatchMessageIdImpl) messageIds.get(i)).toMessageIdImpl(), secondEntryMessageId);
+            assertEquals(MessageIdAdvUtils.discardBatch(messageIds.get(i)), secondEntryMessageId);
         }
 
         assertTrue(interceptor.individualAckedMessageIdList.isEmpty());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
index ceb5c51e6aa77..375bbff8a4df4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
@@ -118,9 +118,6 @@ public void producerSendAsync(TopicType topicType) throws PulsarClientException,
             Message message = consumer.receive();
             assertEquals(new String(message.getData()), messagePrefix + i);
             MessageId messageId = message.getMessageId();
-            if (topicType == TopicType.PARTITIONED) {
-                messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
-            }
             assertTrue(messageIds.remove(messageId), "Failed to receive message");
         }
         log.info("Remaining message IDs = {}", messageIds);
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
new file mode 100644
index 0000000000000..73ecfed0ad059
--- /dev/null
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import java.util.BitSet;
+
+/**
+ * The {@link MessageId} interface provided for advanced users.
+ * 

+ * All built-in MessageId implementations should be able to be cast to MessageIdAdv.
+ * 

+ */
+public interface MessageIdAdv extends MessageId {
+
+    /**
+     * Get the ledger ID.
+     *
+     * @return the ledger ID
+     */
+    long getLedgerId();
+
+    /**
+     * Get the entry ID.
+     *
+     * @return the entry ID
+     */
+    long getEntryId();
+
+    /**
+     * Get the partition index.
+     *
+     * @return -1 if the message is from a non-partitioned topic, otherwise the non-negative partition index
+     */
+    default int getPartitionIndex() {
+        return -1;
+    }
+
+    /**
+     * Get the batch index.
+     *
+     * @return -1 if the message is not in a batch
+     */
+    default int getBatchIndex() {
+        return -1;
+    }
+
+    /**
+     * Get the batch size.
+     *
+     * @return 0 if the message is not in a batch
+     */
+    default int getBatchSize() {
+        return 0;
+    }
+
+    /**
+     * Get the BitSet that indicates which messages in the batch.
+     *
+     * @implNote The message IDs of a batch should share a BitSet. For example, given 3 messages in the same batch whose
+     * size is 3, all message IDs of them should return "111" (i.e. a BitSet whose size is 3 and all bits are 1). If the
+     * 1st message has been acknowledged, the returned BitSet should become "011" (i.e. the 1st bit become 0).
+     *
+     * @return null if the message is a non-batched message
+     */
+    default BitSet getAckSet() {
+        return null;
+    }
+
+    /**
+     * Get the message ID of the first chunk if the current message ID represents the position of a chunked message.
+     *
+     * @implNote A chunked message is distributed across different BookKeeper entries. The message ID of a chunked
+     * message is composed of two message IDs that represent positions of the first and the last chunk. The message ID
+     * itself represents the position of the last chunk.
+     *
+     * @return null if the message is not a chunked message
+     */
+    default MessageIdAdv getFirstChunkMessageId() {
+        return null;
+    }
+
+    /**
+     * The default implementation of {@link Comparable#compareTo(Object)}.
+     */
+    default int compareTo(MessageId o) {
+        if (!(o instanceof MessageIdAdv)) {
+            throw new UnsupportedOperationException("Unknown MessageId type: "
+                    + ((o != null) ? o.getClass().getName() : "null"));
+        }
+        final MessageIdAdv other = (MessageIdAdv) o;
+        int result = Long.compare(this.getLedgerId(), other.getLedgerId());
+        if (result != 0) {
+            return result;
+        }
+        result = Long.compare(this.getEntryId(), other.getEntryId());
+        if (result != 0) {
+            return result;
+        }
+        // TODO: Correct the following compare logics, see https://github.com/apache/pulsar/pull/18981
+        result = Integer.compare(this.getPartitionIndex(), other.getPartitionIndex());
+        if (result != 0) {
+            return result;
+        }
+        return Integer.compare(this.getBatchIndex(), other.getBatchIndex());
+    }
+}
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
index f6109d5f8e87e..b70267bb0fb8b 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.client.api;
 
+import java.util.BitSet;
+
 /**
  * The MessageId used for a consumer that subscribes multiple topics or partitioned topics.
  *
@@ -49,13 +51,13 @@ static TopicMessageId create(String topic, MessageId messageId) {
     /**
      * The simplest implementation of a TopicMessageId interface.
      */
-    class Impl implements TopicMessageId {
+    class Impl implements MessageIdAdv, TopicMessageId {
         private final String topic;
-        private final MessageId messageId;
+        private final MessageIdAdv messageId;
 
         public Impl(String topic, MessageId messageId) {
             this.topic = topic;
-            this.messageId = messageId;
+            this.messageId = (MessageIdAdv) messageId;
         }
 
         @Override
@@ -68,6 +70,41 @@ public String getOwnerTopic() {
             return topic;
         }
 
+        @Override
+        public long getLedgerId() {
+            return messageId.getLedgerId();
+        }
+
+        @Override
+        public long getEntryId() {
+            return messageId.getEntryId();
+        }
+
+        @Override
+        public int getPartitionIndex() {
+            return messageId.getPartitionIndex();
+        }
+
+        @Override
+        public int getBatchIndex() {
+            return messageId.getBatchIndex();
+        }
+
+        @Override
+        public int getBatchSize() {
+            return messageId.getBatchSize();
+        }
+
+        @Override
+        public BitSet getAckSet() {
+            return messageId.getAckSet();
+        }
+
+        @Override
+        public MessageIdAdv getFirstChunkMessageId() {
+            return messageId.getFirstChunkMessageId();
+        }
+
         @Override
         public int compareTo(MessageId o) {
             return messageId.compareTo(o);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java
index d46af1a99e7f0..60d7135e5e4ae 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java
@@ -31,7 +31,7 @@ public interface AcknowledgmentsGroupingTracker extends AutoCloseable {
 
     boolean isDuplicate(MessageId messageId);
 
-    CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, Map properties);
+    CompletableFuture addAcknowledgment(MessageId msgId, AckType ackType, Map properties);
 
     CompletableFuture addListAcknowledgment(List messageIds, AckType ackType,
                                                   Map properties);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java
deleted file mode 100644
index 1c9b66fd2bad5..0000000000000
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import java.util.BitSet;
-
-public class BatchMessageAcker {
-
-    private BatchMessageAcker() {
-        this.bitSet = new BitSet();
-        this.batchSize = 0;
-    }
-
-    static BatchMessageAcker newAcker(int batchSize) {
-        BitSet bitSet = new BitSet(batchSize);
-        bitSet.set(0, batchSize);
-        return new BatchMessageAcker(bitSet, batchSize);
-    }
-
-    // Use the param bitSet as the BatchMessageAcker's bitSet, don't care about the batchSize.
-    static BatchMessageAcker newAcker(BitSet bitSet) {
-        return new BatchMessageAcker(bitSet, -1);
-    }
-
-    // bitset shared across messages in the same batch.
-    private final int batchSize;
-    private final BitSet bitSet;
-    private volatile boolean prevBatchCumulativelyAcked = false;
-
-    BatchMessageAcker(BitSet bitSet, int batchSize) {
-        this.bitSet = bitSet;
-        this.batchSize = batchSize;
-    }
-
-    BitSet getBitSet() {
-        return bitSet;
-    }
-
-    public synchronized int getBatchSize() {
-        return batchSize;
-    }
-
-    public synchronized boolean ackIndividual(int batchIndex) {
-        bitSet.clear(batchIndex);
-        return bitSet.isEmpty();
-    }
-
-    public synchronized int getBitSetSize() {
-        return bitSet.size();
-    }
-
-    public synchronized boolean ackCumulative(int batchIndex) {
-        // +1 since to argument is exclusive
-        bitSet.clear(0, batchIndex + 1);
-        return bitSet.isEmpty();
-    }
-
-    // debug purpose
-    public synchronized int getOutstandingAcks() {
-        return bitSet.cardinality();
-    }
-
-    public void setPrevBatchCumulativelyAcked(boolean acked) {
-        this.prevBatchCumulativelyAcked = acked;
-    }
-
-    public boolean isPrevBatchCumulativelyAcked() {
-        return prevBatchCumulativelyAcked;
-    }
-
-    @Override
-    public String toString() {
-        return "BatchMessageAcker{"
-                + "batchSize=" + batchSize
-                + ", bitSet=" + bitSet
-                + ", prevBatchCumulativelyAcked=" + prevBatchCumulativelyAcked
-                + '}';
-    }
-}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java
deleted file mode 100644
index b70c928b29650..0000000000000
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import java.util.BitSet;
-
-class BatchMessageAckerDisabled extends BatchMessageAcker {
-
-    static final BatchMessageAckerDisabled INSTANCE = new BatchMessageAckerDisabled();
-
-    private BatchMessageAckerDisabled() {
-        super(new BitSet(), 0);
-    }
-
-    @Override
-    public synchronized int getBatchSize() {
-        return 0;
-    }
-
-    @Override
-    public boolean ackIndividual(int batchIndex) {
-        return true;
-    }
-
-    @Override
-    public boolean ackCumulative(int batchIndex) {
-        return true;
-    }
-
-    @Override
-    public int getOutstandingAcks() {
-        return 0;
-    }
-}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
index ed28082ff6a30..e9cddeb65d7f2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
@@ -18,20 +18,16 @@
  */
 package org.apache.pulsar.client.impl;
 
-import javax.annotation.Nonnull;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.TopicMessageId;
+import java.util.BitSet;
+import org.apache.pulsar.client.api.MessageIdAdv;
 
-/**
- */
 public class BatchMessageIdImpl extends MessageIdImpl {
 
     private static final long serialVersionUID = 1L;
-    static final int NO_BATCH = -1;
     private final int batchIndex;
     private final int batchSize;
 
-    private final transient BatchMessageAcker acker;
+    private final BitSet ackSet;
 
     // Private constructor used only for json deserialization
     @SuppressWarnings("unused")
@@ -40,59 +36,35 @@ private BatchMessageIdImpl() {
     }
 
     public BatchMessageIdImpl(long ledgerId, long entryId, int partitionIndex, int batchIndex) {
-        this(ledgerId, entryId, partitionIndex, batchIndex, 0, BatchMessageAckerDisabled.INSTANCE);
+        this(ledgerId, entryId, partitionIndex, batchIndex, 0, null);
     }
 
     public BatchMessageIdImpl(long ledgerId, long entryId, int partitionIndex, int batchIndex, int batchSize,
-                              BatchMessageAcker acker) {
+                              BitSet ackSet) {
         super(ledgerId, entryId, partitionIndex);
         this.batchIndex = batchIndex;
         this.batchSize = batchSize;
-        this.acker = acker;
+        this.ackSet = ackSet;
     }
 
-    public BatchMessageIdImpl(MessageIdImpl other) {
-        super(other.ledgerId, other.entryId, other.partitionIndex);
-        if (other instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl otherId = (BatchMessageIdImpl) other;
-            this.batchIndex = otherId.batchIndex;
-            this.batchSize = otherId.batchSize;
-            this.acker = otherId.acker;
-        } else {
-            this.batchIndex = NO_BATCH;
-            this.batchSize = 0;
-            this.acker = BatchMessageAckerDisabled.INSTANCE;
-        }
+    public BatchMessageIdImpl(MessageIdAdv other) {
+        this(other.getLedgerId(), other.getEntryId(), other.getPartitionIndex(),
+                other.getBatchIndex(), other.getBatchSize(), other.getAckSet());
     }
 
+    @Override
     public int getBatchIndex() {
         return batchIndex;
     }
 
-    @Override
-    public int compareTo(@Nonnull MessageId o) {
-        if (o instanceof MessageIdImpl) {
-            MessageIdImpl other = (MessageIdImpl) o;
-            int batchIndex = (o instanceof BatchMessageIdImpl) ? ((BatchMessageIdImpl) o).batchIndex : NO_BATCH;
-            return messageIdCompare(
-                this.ledgerId, this.entryId, this.partitionIndex, this.batchIndex,
-                other.ledgerId, other.entryId, other.partitionIndex, batchIndex
-            );
-        } else if (o instanceof TopicMessageId) {
-            return compareTo(MessageIdImpl.convertToMessageIdImpl(o));
-        } else {
-            throw new UnsupportedOperationException("Unknown MessageId type: " + o.getClass().getName());
-        }
-    }
-
     @Override
     public int hashCode() {
-        return messageIdHashCode(ledgerId, entryId, partitionIndex, batchIndex);
+        return MessageIdAdvUtils.hashCode(this);
     }
 
     @Override
     public boolean equals(Object o) {
-        return super.equals(o);
+        return MessageIdAdvUtils.equals(this, o);
     }
 
     @Override
@@ -106,39 +78,51 @@ public byte[] toByteArray() {
         return toByteArray(batchIndex, batchSize);
     }
 
+    @Deprecated
     public boolean ackIndividual() {
-        return acker.ackIndividual(batchIndex);
+        return MessageIdAdvUtils.acknowledge(this, true);
     }
 
+    @Deprecated
     public boolean ackCumulative() {
-        return acker.ackCumulative(batchIndex);
+        return MessageIdAdvUtils.acknowledge(this, false);
     }
 
+    @Deprecated
     public int getOutstandingAcksInSameBatch() {
-        return acker.getOutstandingAcks();
+        return 0;
     }
 
+    @Override
     public int getBatchSize() {
-        return acker.getBatchSize();
+        return batchSize;
     }
 
+    @Deprecated
     public int getOriginalBatchSize() {
         return this.batchSize;
     }
 
+    @Deprecated
     public MessageIdImpl prevBatchMessageId() {
-        return new MessageIdImpl(
-            ledgerId, entryId - 1, partitionIndex);
+        return (MessageIdImpl) MessageIdAdvUtils.prevMessageId(this);
     }
 
     // MessageIdImpl is widely used as the key of a hash map, in this case, we should convert the batch message id to
     // have the correct hash code.
+    @Deprecated
     public MessageIdImpl toMessageIdImpl() {
-        return new MessageIdImpl(ledgerId, entryId, partitionIndex);
+        return (MessageIdImpl) MessageIdAdvUtils.discardBatch(this);
     }
 
-    public BatchMessageAcker getAcker() {
-        return acker;
+    @Override
+    public BitSet getAckSet() {
+        return ackSet;
     }
 
+    static BitSet newAckSet(int batchSize) {
+        final BitSet ackSet = new BitSet(batchSize);
+        ackSet.set(0, batchSize);
+        return ackSet;
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java
index 28d5047c8ef25..29ce160442a3b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java
@@ -21,10 +21,10 @@
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import java.util.Objects;
-import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.common.api.proto.MessageIdData;
 
-public class ChunkMessageIdImpl extends MessageIdImpl implements MessageId {
+public class ChunkMessageIdImpl extends MessageIdImpl {
     private final MessageIdImpl firstChunkMsgId;
 
     public ChunkMessageIdImpl(MessageIdImpl firstChunkMsgId, MessageIdImpl lastChunkMsgId) {
@@ -32,11 +32,12 @@ public ChunkMessageIdImpl(MessageIdImpl firstChunkMsgId, MessageIdImpl lastChunk
         this.firstChunkMsgId = firstChunkMsgId;
     }
 
-    public MessageIdImpl getFirstChunkMessageId() {
+    @Override
+    public MessageIdAdv getFirstChunkMessageId() {
         return firstChunkMsgId;
     }
 
-    public MessageIdImpl getLastChunkMessageId() {
+    public MessageIdAdv getLastChunkMessageId() {
         return this;
     }
 
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
index 75d3b2edf6e28..973b3302f4199 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
@@ -47,6 +47,7 @@
 import org.apache.pulsar.client.api.ConsumerEventListener;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.PulsarClientException;
@@ -82,7 +83,7 @@ public abstract class ConsumerBase extends HandlerState implements Consumer> incomingMessages;
-    protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap;
+    protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap;
     protected final ConcurrentLinkedQueue>> pendingReceives;
     protected final int maxReceiverQueueSize;
     private volatile int currentReceiverQueueSize;
@@ -128,7 +129,7 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat
         // Always use growable queue since items can exceed the advertised size
         this.incomingMessages = new GrowableArrayBlockingQueue<>();
         this.unAckedChunkedMessageIdSequenceMap =
-                ConcurrentOpenHashMap.newBuilder().build();
+                ConcurrentOpenHashMap.newBuilder().build();
         this.executorProvider = executorProvider;
         this.externalPinnedExecutor = executorProvider.getExecutor();
         this.internalPinnedExecutor = client.getInternalExecutorService();
@@ -223,14 +224,6 @@ protected void trackUnAckedMsgIfNoListener(MessageId messageId, int redeliveryCo
         }
     }
 
-    protected MessageId normalizeMessageId(MessageId messageId) {
-        if (messageId instanceof BatchMessageIdImpl) {
-            // do not add each item in batch message into tracker
-            return ((BatchMessageIdImpl) messageId).toMessageIdImpl();
-        }
-        return messageId;
-    }
-
     protected void reduceCurrentReceiverQueueSize() {
         if (!conf.isAutoScaledReceiverQueueSizeEnabled()) {
             return;
@@ -1131,7 +1124,7 @@ protected void callMessageListener(Message msg) {
                                 ? ((TopicMessageImpl) msg).getMessage() : msg));
             MessageId id;
             if (this instanceof ConsumerImpl) {
-                id = normalizeMessageId(msg.getMessageId());
+                id = MessageIdAdvUtils.discardBatch(msg.getMessageId());
             } else {
                 id = msg.getMessageId();
             }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index beaa34bf20520..1feef6ca0a642 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -35,6 +35,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -71,6 +72,7 @@
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageCrypto;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClientException;
@@ -154,12 +156,12 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle
     @Getter(AccessLevel.PACKAGE)
     private final int priorityLevel;
     private final SubscriptionMode subscriptionMode;
-    private volatile BatchMessageIdImpl startMessageId;
+    private volatile MessageIdAdv startMessageId;
 
-    private volatile BatchMessageIdImpl seekMessageId;
+    private volatile MessageIdAdv seekMessageId;
     private final AtomicBoolean duringSeek;
 
-    private final BatchMessageIdImpl initialStartMessageId;
+    private final MessageIdAdv initialStartMessageId;
 
     private final long startMessageRollbackDurationInSec;
 
@@ -178,7 +180,7 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle
     private final TopicName topicName;
     private final String topicNameWithoutPartition;
 
-    private final Map>> possibleSendToDeadLetterTopicMessages;
+    private final Map>> possibleSendToDeadLetterTopicMessages;
 
     private final DeadLetterPolicy deadLetterPolicy;
 
@@ -258,12 +260,8 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat
         this.consumerId = client.newConsumerId();
         this.subscriptionMode = conf.getSubscriptionMode();
         if (startMessageId != null) {
-            if (startMessageId instanceof ChunkMessageIdImpl) {
-                this.startMessageId = new BatchMessageIdImpl(
-                        ((ChunkMessageIdImpl) startMessageId).getFirstChunkMessageId());
-            } else {
-                this.startMessageId = new BatchMessageIdImpl((MessageIdImpl) startMessageId);
-            }
+            MessageIdAdv firstChunkMessageId = ((MessageIdAdv) startMessageId).getFirstChunkMessageId();
+            this.startMessageId = (firstChunkMessageId == null) ? (MessageIdAdv) startMessageId : firstChunkMessageId;
         }
         this.initialStartMessageId = this.startMessageId;
         this.startMessageRollbackDurationInSec = startMessageRollbackDurationInSec;
@@ -535,7 +533,6 @@ protected CompletableFuture> internalBatchReceiveAsync() {
     protected CompletableFuture doAcknowledge(MessageId messageId, AckType ackType,
                                                     Map properties,
                                                     TransactionImpl txn) {
-        messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
         if (getState() != State.Ready && getState() != State.Connecting) {
             stats.incrementNumAcksFailed();
             PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState());
@@ -551,16 +548,12 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack
             return doTransactionAcknowledgeForResponse(messageId, ackType, null, properties,
                     new TxnID(txn.getTxnIdMostBits(), txn.getTxnIdLeastBits()));
         }
-        return acknowledgmentsGroupingTracker.addAcknowledgment((MessageIdImpl) messageId, ackType, properties);
+        return acknowledgmentsGroupingTracker.addAcknowledgment(messageId, ackType, properties);
     }
 
     @Override
     protected CompletableFuture doAcknowledge(List messageIdList, AckType ackType,
                                                     Map properties, TransactionImpl txn) {
-
-        for (MessageId messageId : messageIdList) {
-            checkArgument(messageId instanceof MessageIdImpl);
-        }
         if (getState() != State.Ready && getState() != State.Connecting) {
             stats.incrementNumAcksFailed();
             PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState());
@@ -572,7 +565,7 @@ protected CompletableFuture doAcknowledge(List messageIdList, A
             return FutureUtil.failedFuture(exception);
         }
         if (txn != null) {
-            return doTransactionAcknowledgeForResponse(messageIdList, ackType, null,
+            return doTransactionAcknowledgeForResponse(messageIdList, ackType,
                     properties, new TxnID(txn.getTxnIdMostBits(), txn.getTxnIdLeastBits()));
         } else {
             return this.acknowledgmentsGroupingTracker.addListAcknowledgment(messageIdList, ackType, properties);
@@ -592,7 +585,6 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a
                     .InvalidMessageException("Cannot handle message with null messageId"));
         }
 
-        messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
         if (getState() != State.Ready && getState() != State.Connecting) {
             stats.incrementNumAcksFailed();
             PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState());
@@ -922,7 +914,7 @@ protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize
      * Clear the internal receiver queue and returns the message id of what was the 1st message in the queue that was
      * not seen by the application.
      */
-    private BatchMessageIdImpl clearReceiverQueue() {
+    private MessageIdAdv clearReceiverQueue() {
         List> currentMessageQueue = new ArrayList<>(incomingMessages.size());
         incomingMessages.drainTo(currentMessageQueue);
         resetIncomingMessageSize();
@@ -934,17 +926,16 @@ private BatchMessageIdImpl clearReceiverQueue() {
         }
 
         if (!currentMessageQueue.isEmpty()) {
-            MessageIdImpl nextMessageInQueue = (MessageIdImpl) currentMessageQueue.get(0).getMessageId();
-            BatchMessageIdImpl previousMessage;
-            if (nextMessageInQueue instanceof BatchMessageIdImpl) {
+            MessageIdAdv nextMessageInQueue = (MessageIdAdv) currentMessageQueue.get(0).getMessageId();
+            MessageIdAdv previousMessage;
+            if (MessageIdAdvUtils.isBatch(nextMessageInQueue)) {
                 // Get on the previous message within the current batch
                 previousMessage = new BatchMessageIdImpl(nextMessageInQueue.getLedgerId(),
                         nextMessageInQueue.getEntryId(), nextMessageInQueue.getPartitionIndex(),
-                        ((BatchMessageIdImpl) nextMessageInQueue).getBatchIndex() - 1);
+                        nextMessageInQueue.getBatchIndex() - 1);
             } else {
                 // Get on previous message in previous entry
-                previousMessage = new BatchMessageIdImpl(nextMessageInQueue.getLedgerId(),
-                        nextMessageInQueue.getEntryId() - 1, nextMessageInQueue.getPartitionIndex(), -1);
+                previousMessage = MessageIdAdvUtils.prevMessageId(nextMessageInQueue);
             }
             // release messages if they are pooled messages
             currentMessageQueue.forEach(Message::release);
@@ -1126,7 +1117,7 @@ protected  MessageImpl newSingleMessage(final int index,
                                                   final Schema schema,
                                                   final boolean containMetadata,
                                                   final BitSetRecyclable ackBitSet,
-                                                  final BatchMessageAcker acker,
+                                                  final BitSet ackSetInMessageId,
                                                   final int redeliveryCount,
                                                   final long consumerEpoch) {
         if (log.isDebugEnabled()) {
@@ -1161,7 +1152,7 @@ protected  MessageImpl newSingleMessage(final int index,
             }
 
             BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(),
-                    messageId.getEntryId(), getPartitionIndex(), index, numMessages, acker);
+                    messageId.getEntryId(), getPartitionIndex(), index, numMessages, ackSetInMessageId);
 
             final ByteBuf payloadBuffer = (singleMessagePayload != null) ? singleMessagePayload : payload;
             final MessageImpl message = MessageImpl.create(topicName.toString(), batchMessageIdImpl,
@@ -1525,7 +1516,7 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata,
             possibleToDeadLetter = new ArrayList<>();
         }
 
-        BatchMessageAcker acker = BatchMessageAcker.newAcker(batchSize);
+        BitSet ackSetInMessageId = BatchMessageIdImpl.newAckSet(batchSize);
         BitSetRecyclable ackBitSet = null;
         if (ackSet != null && ackSet.size() > 0) {
             ackBitSet = BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet));
@@ -1537,7 +1528,7 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata,
             for (int i = 0; i < batchSize; ++i) {
                 final MessageImpl message = newSingleMessage(i, batchSize, brokerEntryMetadata, msgMetadata,
                         singleMessageMetadata, uncompressedPayload, batchMessage, schema, true,
-                        ackBitSet, acker, redeliveryCount, consumerEpoch);
+                        ackBitSet, ackSetInMessageId, redeliveryCount, consumerEpoch);
                 if (message == null) {
                     skippedMessages++;
                     continue;
@@ -1634,7 +1625,7 @@ protected void trackMessage(MessageId messageId) {
 
     protected void trackMessage(MessageId messageId, int redeliveryCount) {
         if (conf.getAckTimeoutMillis() > 0 && messageId instanceof MessageIdImpl) {
-            MessageId id = normalizeMessageId(messageId);
+            MessageId id = MessageIdAdvUtils.discardBatch(messageId);
             if (hasParentConsumer) {
                 //TODO: check parent consumer here
                 // we should no longer track this message, TopicsConsumer will take care from now onwards
@@ -1931,8 +1922,6 @@ public void redeliverUnacknowledgedMessages(Set messageIds) {
             return;
         }
 
-        checkArgument(messageIds.stream().findFirst().get() instanceof MessageIdImpl);
-
         if (conf.getSubscriptionType() != SubscriptionType.Shared
                 && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
             // We cannot redeliver single messages if subscription type is not Shared
@@ -1942,11 +1931,7 @@ public void redeliverUnacknowledgedMessages(Set messageIds) {
         ClientCnx cnx = cnx();
         if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getValue()) {
             int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds);
-            Iterable> batches = Iterables.partition(
-                messageIds.stream()
-                    .map(messageId -> (MessageIdImpl) messageId)
-                    .collect(Collectors.toSet()), MAX_REDELIVER_UNACKNOWLEDGED);
-            batches.forEach(ids -> {
+            Iterables.partition(messageIds, MAX_REDELIVER_UNACKNOWLEDGED).forEach(ids -> {
                 getRedeliveryMessageIdData(ids).thenAccept(messageIdData -> {
                     if (!messageIdData.isEmpty()) {
                         ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdData);
@@ -1985,11 +1970,12 @@ protected void completeOpBatchReceive(OpBatchReceive op) {
         notifyPendingBatchReceivedCallBack(op.future);
     }
 
-    private CompletableFuture> getRedeliveryMessageIdData(List messageIds) {
+    private CompletableFuture> getRedeliveryMessageIdData(List messageIds) {
         if (messageIds == null || messageIds.isEmpty()) {
             return CompletableFuture.completedFuture(Collections.emptyList());
         }
-        List> futures = messageIds.stream().map(messageId -> {
+        List> futures = messageIds.stream().map(originalMessageId -> {
+            final MessageIdAdv messageId = (MessageIdAdv) originalMessageId;
             CompletableFuture future = processPossibleToDLQ(messageId);
             return future.thenApply(sendToDLQ -> {
                 if (!sendToDLQ) {
@@ -2005,20 +1991,15 @@ private CompletableFuture> getRedeliveryMessageIdData(List processPossibleToDLQ(MessageIdImpl messageId) {
+    private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) {
         List> deadLetterMessages = null;
         if (possibleSendToDeadLetterTopicMessages != null) {
-            if (messageId instanceof BatchMessageIdImpl) {
-                messageId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(),
-                        getPartitionIndex());
-            }
-            deadLetterMessages = possibleSendToDeadLetterTopicMessages.get(messageId);
+            deadLetterMessages = possibleSendToDeadLetterTopicMessages.get(MessageIdAdvUtils.discardBatch(messageId));
         }
         CompletableFuture result = new CompletableFuture<>();
         if (deadLetterMessages != null) {
             initDeadLetterProducerIfNeeded();
             List> finalDeadLetterMessages = deadLetterMessages;
-            MessageIdImpl finalMessageId = messageId;
             deadLetterProducer.thenAcceptAsync(producerDLQ -> {
                 for (MessageImpl message : finalDeadLetterMessages) {
                     String originMessageIdStr = message.getMessageId().toString();
@@ -2032,12 +2013,12 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId)
                     }
                     typedMessageBuilderNew.sendAsync()
                             .thenAccept(messageIdInDLQ -> {
-                                possibleSendToDeadLetterTopicMessages.remove(finalMessageId);
-                                acknowledgeAsync(finalMessageId).whenComplete((v, ex) -> {
+                                possibleSendToDeadLetterTopicMessages.remove(messageId);
+                                acknowledgeAsync(messageId).whenComplete((v, ex) -> {
                                     if (ex != null) {
                                         log.warn("[{}] [{}] [{}] Failed to acknowledge the message {} of the original"
                                                         + " topic but send to the DLQ successfully.",
-                                                topicName, subscription, consumerName, finalMessageId, ex);
+                                                topicName, subscription, consumerName, messageId, ex);
                                         result.complete(false);
                                     } else {
                                         result.complete(true);
@@ -2047,11 +2028,11 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId)
                                 if (ex instanceof PulsarClientException.ProducerQueueIsFullError) {
                                     log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}: {}",
                                             topicName, subscription, consumerName,
-                                            deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex.getMessage());
+                                            deadLetterPolicy.getDeadLetterTopic(), messageId, ex.getMessage());
                                 } else {
                                     log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}",
                                             topicName, subscription, consumerName,
-                                            deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex);
+                                            deadLetterPolicy.getDeadLetterTopic(), messageId, ex);
                                 }
                                 result.complete(false);
                                 return null;
@@ -2154,8 +2135,8 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek,
         final CompletableFuture seekFuture = new CompletableFuture<>();
         ClientCnx cnx = cnx();
 
-        BatchMessageIdImpl originSeekMessageId = seekMessageId;
-        seekMessageId = new BatchMessageIdImpl((MessageIdImpl) seekId);
+        MessageIdAdv originSeekMessageId = seekMessageId;
+        seekMessageId = (MessageIdAdv) seekId;
         duringSeek.set(true);
         log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy);
 
@@ -2193,29 +2174,28 @@ public CompletableFuture seekAsync(long timestamp) {
     }
 
     @Override
-    public CompletableFuture seekAsync(MessageId originalMessageId) {
-        final MessageIdImpl messageId = MessageIdImpl.convertToMessageIdImpl(originalMessageId);
+    public CompletableFuture seekAsync(MessageId messageId) {
         String seekBy = String.format("the message %s", messageId.toString());
         return seekAsyncCheckState(seekBy).orElseGet(() -> {
             long requestId = client.newRequestId();
-            ByteBuf seek = null;
-            if (messageId instanceof BatchMessageIdImpl) {
-                BatchMessageIdImpl msgId = (BatchMessageIdImpl) messageId;
-                // Initialize ack set
-                BitSetRecyclable ackSet = BitSetRecyclable.create();
-                ackSet.set(0, msgId.getBatchSize());
-                ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0));
-                long[] ackSetArr = ackSet.toLongArray();
-                ackSet.recycle();
-
-                seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr);
-            } else if (messageId instanceof ChunkMessageIdImpl) {
-                ChunkMessageIdImpl msgId = (ChunkMessageIdImpl) messageId;
-                seek = Commands.newSeek(consumerId, requestId, msgId.getFirstChunkMessageId().getLedgerId(),
-                        msgId.getFirstChunkMessageId().getEntryId(), new long[0]);
+            final MessageIdAdv msgId = (MessageIdAdv) messageId;
+            final MessageIdAdv firstChunkMsgId = msgId.getFirstChunkMessageId();
+            final ByteBuf seek;
+            if (msgId.getFirstChunkMessageId() != null) {
+                seek = Commands.newSeek(consumerId, requestId, firstChunkMsgId.getLedgerId(),
+                        firstChunkMsgId.getEntryId(), new long[0]);
             } else {
-                seek = Commands.newSeek(
-                        consumerId, requestId, messageId.getLedgerId(), messageId.getEntryId(), new long[0]);
+                final long[] ackSetArr;
+                if (MessageIdAdvUtils.isBatch(msgId)) {
+                    final BitSetRecyclable ackSet = BitSetRecyclable.create();
+                    ackSet.set(0, msgId.getBatchSize());
+                    ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0));
+                    ackSetArr = ackSet.toLongArray();
+                    ackSet.recycle();
+                } else {
+                    ackSetArr = new long[0];
+                }
+                seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr);
             }
             return seekAsyncInternal(requestId, seek, messageId, seekBy);
         });
@@ -2247,9 +2227,8 @@ public CompletableFuture hasMessageAvailableAsync() {
                 }
 
                 future.thenAccept(response -> {
-                    MessageIdImpl lastMessageId = MessageIdImpl.convertToMessageIdImpl(response.lastMessageId);
-                    MessageIdImpl markDeletePosition = MessageIdImpl
-                            .convertToMessageIdImpl(response.markDeletePosition);
+                    MessageIdAdv lastMessageId = (MessageIdAdv) response.lastMessageId;
+                    MessageIdAdv markDeletePosition = (MessageIdAdv) response.markDeletePosition;
 
                     if (markDeletePosition != null && !(markDeletePosition.getEntryId() < 0
                             && markDeletePosition.getLedgerId() > lastMessageId.getLedgerId())) {
@@ -2434,16 +2413,6 @@ private void internalGetLastMessageIdAsync(final Backoff backoff,
         }
     }
 
-    private MessageIdImpl getMessageIdImpl(Message msg) {
-        MessageIdImpl messageId = (MessageIdImpl) msg.getMessageId();
-        if (messageId instanceof BatchMessageIdImpl) {
-            // messageIds contain MessageIdImpl, not BatchMessageIdImpl
-            messageId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), getPartitionIndex());
-        }
-        return messageId;
-    }
-
-
     private boolean isMessageUndecryptable(MessageMetadata msgMetadata) {
         return (msgMetadata.getEncryptionKeysCount() > 0 && conf.getCryptoKeyReader() == null
                 && conf.getCryptoFailureAction() == ConsumerCryptoFailureAction.CONSUME);
@@ -2486,7 +2455,7 @@ private int removeExpiredMessagesFromQueue(Set messageIds) {
         int messagesFromQueue = 0;
         Message peek = incomingMessages.peek();
         if (peek != null) {
-            MessageIdImpl messageId = getMessageIdImpl(peek);
+            MessageIdAdv messageId = MessageIdAdvUtils.discardBatch(peek.getMessageId());
             if (!messageIds.contains(messageId)) {
                 // first message is not expired, then no message is expired in queue.
                 return 0;
@@ -2497,7 +2466,7 @@ private int removeExpiredMessagesFromQueue(Set messageIds) {
             while (message != null) {
                 decreaseIncomingMessageSize(message);
                 messagesFromQueue++;
-                MessageIdImpl id = getMessageIdImpl(message);
+                MessageIdAdv id = MessageIdAdvUtils.discardBatch(message.getMessageId());
                 if (!messageIds.contains(id)) {
                     messageIds.add(id);
                     break;
@@ -2691,33 +2660,26 @@ private void removeChunkMessage(String msgUUID, ChunkedMessageCtx chunkedMsgCtx,
     private CompletableFuture doTransactionAcknowledgeForResponse(MessageId messageId, AckType ackType,
                                                                         ValidationError validationError,
                                                                         Map properties, TxnID txnID) {
-        BitSetRecyclable bitSetRecyclable = null;
-        long ledgerId;
-        long entryId;
-        ByteBuf cmd;
         long requestId = client.newRequestId();
-        if (messageId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-            bitSetRecyclable = BitSetRecyclable.create();
-            ledgerId = batchMessageId.getLedgerId();
-            entryId = batchMessageId.getEntryId();
+        final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+        final long ledgerId = messageIdAdv.getLedgerId();
+        final long entryId = messageIdAdv.getEntryId();
+        final ByteBuf cmd;
+        if (MessageIdAdvUtils.isBatch(messageIdAdv)) {
+            BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create();
+            bitSetRecyclable.set(0, messageIdAdv.getBatchSize());
             if (ackType == AckType.Cumulative) {
-                batchMessageId.ackCumulative();
-                bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                bitSetRecyclable.clear(0, batchMessageId.getBatchIndex() + 1);
+                MessageIdAdvUtils.acknowledge(messageIdAdv, false);
+                bitSetRecyclable.clear(0, messageIdAdv.getBatchIndex() + 1);
             } else {
-                bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                bitSetRecyclable.clear(batchMessageId.getBatchIndex());
+                bitSetRecyclable.clear(messageIdAdv.getBatchIndex());
             }
             cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType, validationError, properties,
-                    txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, batchMessageId.getBatchSize());
+                    txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, messageIdAdv.getBatchSize());
             bitSetRecyclable.recycle();
         } else {
-            MessageIdImpl singleMessage = (MessageIdImpl) messageId;
-            ledgerId = singleMessage.getLedgerId();
-            entryId = singleMessage.getEntryId();
-            cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType,
-                    validationError, properties, txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId);
+            cmd = Commands.newAck(consumerId, ledgerId, entryId, null, ackType, validationError, properties,
+                    txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId);
         }
 
         if (ackType == AckType.Cumulative) {
@@ -2736,58 +2698,42 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me
     }
 
     private CompletableFuture doTransactionAcknowledgeForResponse(List messageIds, AckType ackType,
-                                                                        ValidationError validationError,
                                                                         Map properties, TxnID txnID) {
-        BitSetRecyclable bitSetRecyclable = null;
-        long ledgerId;
-        long entryId;
-        ByteBuf cmd;
         long requestId = client.newRequestId();
         List messageIdDataList = new LinkedList<>();
         for (MessageId messageId : messageIds) {
-            if (messageId instanceof BatchMessageIdImpl) {
-                BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-                bitSetRecyclable = BitSetRecyclable.create();
+            final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+            final MessageIdData messageIdData = new MessageIdData();
+            messageIdData.setLedgerId(messageIdAdv.getLedgerId());
+            messageIdData.setEntryId(messageIdAdv.getEntryId());
+            if (MessageIdAdvUtils.isBatch(messageIdAdv)) {
+                final BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create();
+                bitSetRecyclable.set(0, messageIdAdv.getBatchSize());
                 if (ackType == AckType.Cumulative) {
-                    batchMessageId.ackCumulative();
-                    bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                    bitSetRecyclable.clear(0, batchMessageId.getBatchIndex() + 1);
+                    MessageIdAdvUtils.acknowledge(messageIdAdv, false);
+                    bitSetRecyclable.clear(0, messageIdAdv.getBatchIndex() + 1);
                 } else {
-                    bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                    bitSetRecyclable.clear(batchMessageId.getBatchIndex());
+                    bitSetRecyclable.clear(messageIdAdv.getBatchIndex());
                 }
-                MessageIdData messageIdData = new MessageIdData();
-                messageIdData.setLedgerId(batchMessageId.getLedgerId());
-                messageIdData.setEntryId(batchMessageId.getEntryId());
-                messageIdData.setBatchSize(batchMessageId.getBatchSize());
-                long[] as = bitSetRecyclable.toLongArray();
-                for (int i = 0; i < as.length; i++) {
-                    messageIdData.addAckSet(as[i]);
+                for (long x : bitSetRecyclable.toLongArray()) {
+                    messageIdData.addAckSet(x);
                 }
                 bitSetRecyclable.recycle();
-                messageIdDataList.add(messageIdData);
-            } else {
-                MessageIdImpl singleMessage = (MessageIdImpl) messageId;
-                ledgerId = singleMessage.getLedgerId();
-                entryId = singleMessage.getEntryId();
-                MessageIdData messageIdData = new MessageIdData();
-                messageIdData.setLedgerId(ledgerId);
-                messageIdData.setEntryId(entryId);
-                messageIdDataList.add(messageIdData);
             }
 
+            messageIdDataList.add(messageIdData);
             if (ackType == AckType.Cumulative) {
                 unAckedMessageTracker.removeMessagesTill(messageId);
             } else {
                 unAckedMessageTracker.remove(messageId);
             }
         }
-        cmd = Commands.newAck(consumerId, messageIdDataList, ackType, validationError, properties,
+        final ByteBuf cmd = Commands.newAck(consumerId, messageIdDataList, ackType, null, properties,
                 txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId);
         return cnx().newAckForReceipt(cmd, requestId);
     }
 
-    public Map>> getPossibleSendToDeadLetterTopicMessages() {
+    public Map>> getPossibleSendToDeadLetterTopicMessages() {
         return possibleSendToDeadLetterTopicMessages;
     }
 
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java
new file mode 100644
index 0000000000000..c8b18524ec052
--- /dev/null
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl;
+
+import java.util.BitSet;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
+
+public class MessageIdAdvUtils {
+
+    static int hashCode(MessageIdAdv msgId) {
+        return (int) (31 * (msgId.getLedgerId() + 31 * msgId.getEntryId())
+                + (31 * (long) msgId.getPartitionIndex()) + msgId.getBatchIndex());
+    }
+
+    static boolean equals(MessageIdAdv lhs, Object o) {
+        if (!(o instanceof MessageIdAdv)) {
+            return false;
+        }
+        final MessageIdAdv rhs = (MessageIdAdv) o;
+        return lhs.getLedgerId() == rhs.getLedgerId()
+                && lhs.getEntryId() == rhs.getEntryId()
+                && lhs.getPartitionIndex() == rhs.getPartitionIndex()
+                && lhs.getBatchIndex() == rhs.getBatchIndex();
+    }
+
+    static boolean acknowledge(MessageIdAdv msgId, boolean individual) {
+        if (!isBatch(msgId)) {
+            return true;
+        }
+        final BitSet ackSet = msgId.getAckSet();
+        if (ackSet == null) {
+            // The internal MessageId implementation should never reach here. If users have implemented their own
+            // MessageId and getAckSet() is not override, return false to avoid acknowledge current entry.
+            return false;
+        }
+        int batchIndex = msgId.getBatchIndex();
+        if (individual) {
+            ackSet.clear(batchIndex);
+        } else {
+            ackSet.clear(0, batchIndex + 1);
+        }
+        return ackSet.isEmpty();
+    }
+
+    static boolean isBatch(MessageIdAdv msgId) {
+        return msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0;
+    }
+
+    static MessageIdAdv discardBatch(MessageId messageId) {
+        MessageIdAdv msgId = (MessageIdAdv) messageId;
+        return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId(), msgId.getPartitionIndex());
+    }
+
+    static MessageIdAdv prevMessageId(MessageIdAdv msgId) {
+        return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId() - 1, msgId.getPartitionIndex());
+    }
+}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
index 1a0f491a6a7bb..83ee762578390 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
@@ -18,21 +18,17 @@
  */
 package org.apache.pulsar.client.impl;
 
-import static org.apache.pulsar.client.impl.BatchMessageIdImpl.NO_BATCH;
-import com.google.common.collect.ComparisonChain;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.io.IOException;
 import java.util.Objects;
-import javax.annotation.Nonnull;
 import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.TopicMessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.common.api.proto.MessageIdData;
-import org.apache.pulsar.common.classification.InterfaceStability;
 import org.apache.pulsar.common.naming.TopicName;
 
-public class MessageIdImpl implements MessageId {
+public class MessageIdImpl implements MessageIdAdv {
     protected final long ledgerId;
     protected final long entryId;
     protected final int partitionIndex;
@@ -49,28 +45,29 @@ public MessageIdImpl(long ledgerId, long entryId, int partitionIndex) {
         this.partitionIndex = partitionIndex;
     }
 
+    @Override
     public long getLedgerId() {
         return ledgerId;
     }
 
+    @Override
     public long getEntryId() {
         return entryId;
     }
 
+    @Override
     public int getPartitionIndex() {
         return partitionIndex;
     }
 
     @Override
     public int hashCode() {
-        return messageIdHashCode(ledgerId, entryId, partitionIndex, NO_BATCH);
+        return MessageIdAdvUtils.hashCode(this);
     }
 
     @Override
     public boolean equals(Object o) {
-        return (o instanceof MessageId)
-                && !(o instanceof MultiMessageIdImpl)
-                && (compareTo((MessageId) o) == 0);
+        return MessageIdAdvUtils.equals(this, o);
     }
 
     @Override
@@ -100,7 +97,7 @@ public static MessageId fromByteArray(byte[] data) throws IOException {
         if (idData.hasBatchIndex()) {
             if (idData.hasBatchSize()) {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
-                    idData.getBatchIndex(), idData.getBatchSize(), BatchMessageAcker.newAcker(idData.getBatchSize()));
+                    idData.getBatchIndex(), idData.getBatchSize(), BatchMessageIdImpl.newAckSet(idData.getBatchSize()));
             } else {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
                     idData.getBatchIndex());
@@ -118,22 +115,6 @@ public static MessageId fromByteArray(byte[] data) throws IOException {
         return messageId;
     }
 
-    @InterfaceStability.Unstable
-    public static MessageIdImpl convertToMessageIdImpl(MessageId messageId) {
-        if (messageId instanceof TopicMessageId) {
-            if (messageId instanceof TopicMessageIdImpl) {
-                return (MessageIdImpl) ((TopicMessageIdImpl) messageId).getInnerMessageId();
-            } else {
-                try {
-                    return (MessageIdImpl) MessageId.fromByteArray(messageId.toByteArray());
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-        return (MessageIdImpl) messageId;
-    }
-
     public static MessageId fromByteArrayWithTopic(byte[] data, String topicName) throws IOException {
         return fromByteArrayWithTopic(data, TopicName.get(topicName));
     }
@@ -152,10 +133,10 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName)
             if (idData.hasBatchSize()) {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
                         idData.getBatchIndex(), idData.getBatchSize(),
-                        BatchMessageAcker.newAcker(idData.getBatchSize()));
+                        BatchMessageIdImpl.newAckSet(idData.getBatchSize()));
             } else {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
-                        idData.getBatchIndex(), 0, BatchMessageAckerDisabled.INSTANCE);
+                        idData.getBatchIndex(), 0, null);
             }
         } else {
             messageId = new MessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition());
@@ -207,36 +188,4 @@ public byte[] toByteArray() {
         // there is no message batch so we pass -1
         return toByteArray(-1, 0);
     }
-
-    @Override
-    public int compareTo(@Nonnull MessageId o) {
-        if (o instanceof MessageIdImpl) {
-            MessageIdImpl other = (MessageIdImpl) o;
-            int batchIndex = (o instanceof BatchMessageIdImpl) ? ((BatchMessageIdImpl) o).getBatchIndex() : NO_BATCH;
-            return messageIdCompare(
-                this.ledgerId, this.entryId, this.partitionIndex, NO_BATCH,
-                other.ledgerId, other.entryId, other.partitionIndex, batchIndex
-            );
-        } else if (o instanceof TopicMessageId) {
-            return compareTo(convertToMessageIdImpl(o));
-        } else {
-            throw new UnsupportedOperationException("Unknown MessageId type: " + o.getClass().getName());
-        }
-    }
-
-    static int messageIdHashCode(long ledgerId, long entryId, int partitionIndex, int batchIndex) {
-        return (int) (31 * (ledgerId + 31 * entryId) + (31 * (long) partitionIndex) + batchIndex);
-    }
-
-    static int messageIdCompare(
-        long ledgerId1, long entryId1, int partitionIndex1, int batchIndex1,
-        long ledgerId2, long entryId2, int partitionIndex2, int batchIndex2
-    ) {
-        return ComparisonChain.start()
-            .compare(ledgerId1, ledgerId2)
-            .compare(entryId1, entryId2)
-            .compare(partitionIndex1, partitionIndex2)
-            .compare(batchIndex1, batchIndex2)
-            .result();
-    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
index 0b6fb608ee62b..d369d639a73a0 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
@@ -40,6 +40,7 @@
 import lombok.Getter;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SchemaSerializationException;
 import org.apache.pulsar.client.impl.schema.AbstractSchema;
@@ -714,9 +715,10 @@ public boolean hasIndex() {
     @Override
     public Optional getIndex() {
         if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) {
-            if (msgMetadata.hasNumMessagesInBatch() && messageId instanceof BatchMessageIdImpl) {
-                int batchSize = ((BatchMessageIdImpl) messageId).getBatchSize();
-                int batchIndex = ((BatchMessageIdImpl) messageId).getBatchIndex();
+            MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+            if (msgMetadata.hasNumMessagesInBatch() && MessageIdAdvUtils.isBatch(messageIdAdv)) {
+                int batchSize = messageIdAdv.getBatchSize();
+                int batchIndex = messageIdAdv.getBatchIndex();
                 return Optional.of(brokerEntryMetadata.getIndex() - batchSize + batchIndex + 1);
             }
             return Optional.of(brokerEntryMetadata.getIndex());
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java
index dcae86bd01a3b..f4c9aa2707477 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java
@@ -21,6 +21,7 @@
 import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
 import io.netty.buffer.ByteBuf;
 import io.netty.util.Recycler;
+import java.util.BitSet;
 import java.util.List;
 import lombok.NonNull;
 import org.apache.pulsar.client.api.Message;
@@ -50,7 +51,7 @@ protected MessagePayloadContextImpl newObject(Handle
     private MessageIdImpl messageId;
     private ConsumerImpl consumer;
     private int redeliveryCount;
-    private BatchMessageAcker acker;
+    private BitSet ackSetInMessageId;
     private BitSetRecyclable ackBitSet;
     private long consumerEpoch;
 
@@ -73,7 +74,7 @@ public static MessagePayloadContextImpl get(final BrokerEntryMetadata brokerEntr
         context.messageId = messageId;
         context.consumer = consumer;
         context.redeliveryCount = redeliveryCount;
-        context.acker = BatchMessageAcker.newAcker(context.getNumMessages());
+        context.ackSetInMessageId = BatchMessageIdImpl.newAckSet(context.getNumMessages());
         context.ackBitSet = (ackSet != null && ackSet.size() > 0)
                 ? BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet))
                 : null;
@@ -88,7 +89,7 @@ public void recycle() {
         consumer = null;
         redeliveryCount = 0;
         consumerEpoch = DEFAULT_CONSUMER_EPOCH;
-        acker = null;
+        ackSetInMessageId = null;
         if (ackBitSet != null) {
             ackBitSet.recycle();
             ackBitSet = null;
@@ -134,7 +135,7 @@ public  Message getMessageAt(int index,
                     schema,
                     containMetadata,
                     ackBitSet,
-                    acker,
+                    ackSetInMessageId,
                     redeliveryCount,
                     consumerEpoch);
         } finally {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index f993304b0780a..5fe0e4a82b840 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -55,6 +55,7 @@
 import org.apache.pulsar.client.api.ConsumerStats;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.PulsarClientException.NotSupportedException;
@@ -100,7 +101,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase {
     private final MultiTopicConsumerStatsRecorderImpl stats;
     private final ConsumerConfigurationData internalConfig;
 
-    private volatile BatchMessageIdImpl startMessageId = null;
+    private volatile MessageIdAdv startMessageId;
     private final long startMessageRollbackDurationInSec;
     MultiTopicsConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf,
             ExecutorProvider executorProvider, CompletableFuture> subscribeFuture, Schema schema,
@@ -139,9 +140,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase {
         this.consumers = new ConcurrentHashMap<>();
         this.pausedConsumers = new ConcurrentLinkedQueue<>();
         this.allTopicPartitionsNumber = new AtomicInteger(0);
-        this.startMessageId = startMessageId != null
-                ? new BatchMessageIdImpl(MessageIdImpl.convertToMessageIdImpl(startMessageId))
-                : null;
+        this.startMessageId = (MessageIdAdv) startMessageId;
         this.startMessageRollbackDurationInSec = startMessageRollbackDurationInSec;
         this.paused = conf.isStartPaused();
 
@@ -454,18 +453,15 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack
             return FutureUtil.failedFuture(new PulsarClientException("Consumer already closed"));
         }
 
-        TopicMessageId topicMessageId = (TopicMessageId) messageId;
-        ConsumerImpl consumer = consumers.get(topicMessageId.getOwnerTopic());
+        ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic());
         if (consumer == null) {
             return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException());
         }
-        MessageId innerMessageId = MessageIdImpl.convertToMessageIdImpl(topicMessageId);
         if (ackType == AckType.Cumulative) {
-            return consumer.acknowledgeCumulativeAsync(innerMessageId);
+            return consumer.acknowledgeCumulativeAsync(messageId);
         } else {
-            return consumer.doAcknowledgeWithTxn(innerMessageId, ackType, properties, txnImpl)
-                .thenRun(() ->
-                    unAckedMessageTracker.remove(topicMessageId));
+            return consumer.doAcknowledgeWithTxn(messageId, ackType, properties, txnImpl)
+                .thenRun(() -> unAckedMessageTracker.remove(messageId));
         }
     }
 
@@ -490,10 +486,9 @@ protected CompletableFuture doAcknowledge(List messageIdList,
             }
             Map> topicToMessageIdMap = new HashMap<>();
             for (MessageId messageId : messageIdList) {
-                TopicMessageId topicMessageId = (TopicMessageId) messageId;
-                topicToMessageIdMap.putIfAbsent(topicMessageId.getOwnerTopic(), new ArrayList<>());
-                topicToMessageIdMap.get(topicMessageId.getOwnerTopic())
-                        .add(MessageIdImpl.convertToMessageIdImpl(topicMessageId));
+                String ownerTopic = ((TopicMessageId) messageId).getOwnerTopic();
+                topicToMessageIdMap.putIfAbsent(ownerTopic, new ArrayList<>());
+                topicToMessageIdMap.get(ownerTopic).add(messageId);
             }
             final Map, List> consumerToMessageIds = new IdentityHashMap<>();
             for (Map.Entry> entry : topicToMessageIdMap.entrySet()) {
@@ -549,10 +544,8 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a
     @Override
     public void negativeAcknowledge(MessageId messageId) {
         checkArgument(messageId instanceof TopicMessageId);
-        TopicMessageId topicMessageId = (TopicMessageId) messageId;
-
-        ConsumerImpl consumer = consumers.get(topicMessageId.getOwnerTopic());
-        consumer.negativeAcknowledge(MessageIdImpl.convertToMessageIdImpl(topicMessageId));
+        ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic());
+        consumer.negativeAcknowledge(messageId);
     }
 
     @Override
@@ -705,12 +698,11 @@ public void redeliverUnacknowledgedMessages(Set messageIds) {
             return;
         }
         removeExpiredMessagesFromQueue(messageIds);
-        messageIds.stream().map(messageId -> (TopicMessageId) messageId)
-            .collect(Collectors.groupingBy(TopicMessageId::getOwnerTopic, Collectors.toSet()))
-            .forEach((topicName, messageIds1) ->
-                consumers.get(topicName)
-                    .redeliverUnacknowledgedMessages(messageIds1.stream()
-                        .map(MessageIdImpl::convertToMessageIdImpl).collect(Collectors.toSet())));
+        messageIds.stream()
+                .collect(Collectors.groupingBy(
+                        msgId -> ((TopicMessageIdImpl) msgId).getOwnerTopic(), Collectors.toSet()))
+                .forEach((topicName, messageIds1) ->
+                        consumers.get(topicName).redeliverUnacknowledgedMessages(messageIds1));
         resumeReceivingFromPausedConsumersIfNeeded();
     }
 
@@ -1508,7 +1500,7 @@ public CompletableFuture getLastMessageIdAsync() {
 
     public static boolean isIllegalMultiTopicsMessageId(MessageId messageId) {
         //only support earliest/latest
-        return !MessageId.earliest.equals(messageId) && !MessageId.latest.equals(messageId);
+        return !messageId.equals(MessageId.earliest) && !messageId.equals(MessageId.latest);
     }
 
     public void tryAcknowledgeMessage(Message msg) {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java
index 70d57db3bb691..37f58a0218091 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java
@@ -95,14 +95,6 @@ public synchronized void add(Message message) {
     }
 
     private synchronized void add(MessageId messageId, int redeliveryCount) {
-        messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
-
-        if (messageId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-            messageId = new MessageIdImpl(batchMessageId.getLedgerId(), batchMessageId.getEntryId(),
-                    batchMessageId.getPartitionIndex());
-        }
-
         if (nackedMessages == null) {
             nackedMessages = new HashMap<>();
         }
@@ -113,7 +105,7 @@ private synchronized void add(MessageId messageId, int redeliveryCount) {
         } else {
             backoffNs = nackDelayNanos;
         }
-        nackedMessages.put(messageId, System.nanoTime() + backoffNs);
+        nackedMessages.put(MessageIdAdvUtils.discardBatch(messageId), System.nanoTime() + backoffNs);
 
         if (this.timeout == null) {
             // Schedule a task and group all the redeliveries for same period. Leave a small buffer to allow for
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java
index 32f8fb922304b..e8951cd3d1692 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java
@@ -43,7 +43,7 @@ public boolean isDuplicate(MessageId messageId) {
         return false;
     }
 
-    public CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, Map addAcknowledgment(MessageId msgId, AckType ackType, Map properties) {
         // no-op
         return CompletableFuture.completedFuture(null);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
index fef0bcb8906f1..9086ccc4ef0e0 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
@@ -23,6 +23,7 @@
 import io.netty.channel.EventLoopGroup;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -43,6 +44,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Triple;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
 import org.apache.pulsar.client.util.TimedCompletableFuture;
@@ -79,8 +81,8 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments
      * This is a set of all the individual acks that the application has issued and that were not already sent to
      * broker.
      */
-    private final ConcurrentSkipListSet pendingIndividualAcks;
-    private final ConcurrentHashMap pendingIndividualBatchIndexAcks;
+    private final ConcurrentSkipListSet pendingIndividualAcks;
+    private final ConcurrentHashMap pendingIndividualBatchIndexAcks;
 
     private final ScheduledFuture scheduledTask;
     private final boolean batchIndexAckEnabled;
@@ -113,18 +115,16 @@ public PersistentAcknowledgmentsGroupingTracker(ConsumerImpl consumer, Consum
      */
     @Override
     public boolean isDuplicate(MessageId messageId) {
-        if (!(messageId instanceof MessageIdImpl)) {
+        if (!(messageId instanceof MessageIdAdv)) {
             throw new IllegalArgumentException("isDuplicated cannot accept "
                     + messageId.getClass().getName() + ": " + messageId);
         }
-        if (lastCumulativeAck.compareTo(messageId) >= 0) {
+        final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+        if (lastCumulativeAck.compareTo(messageIdAdv) >= 0) {
             // Already included in a cumulative ack
             return true;
         } else {
-            final MessageIdImpl messageIdImpl = (messageId instanceof BatchMessageIdImpl)
-                    ? ((BatchMessageIdImpl) messageId).toMessageIdImpl()
-                    : (MessageIdImpl) messageId;
-            return pendingIndividualAcks.contains(messageIdImpl);
+            return pendingIndividualAcks.contains(MessageIdAdvUtils.discardBatch(messageIdAdv));
         }
     }
 
@@ -135,10 +135,10 @@ public CompletableFuture addListAcknowledgment(List messageIds,
             if (consumer.isAckReceiptEnabled()) {
                 Set> completableFutureSet = new HashSet<>();
                 messageIds.forEach(messageId ->
-                        completableFutureSet.add(addAcknowledgment((MessageIdImpl) messageId, ackType, properties)));
+                        completableFutureSet.add(addAcknowledgment(messageId, ackType, properties)));
                 return FutureUtil.waitForAll(new ArrayList<>(completableFutureSet));
             } else {
-                messageIds.forEach(messageId -> addAcknowledgment((MessageIdImpl) messageId, ackType, properties));
+                messageIds.forEach(messageId -> addAcknowledgment(messageId, ackType, properties));
                 return CompletableFuture.completedFuture(null);
             }
         } else {
@@ -162,46 +162,43 @@ public CompletableFuture addListAcknowledgment(List messageIds,
 
     private void addListAcknowledgment(List messageIds) {
         for (MessageId messageId : messageIds) {
-            if (messageId instanceof BatchMessageIdImpl) {
-                BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-                addIndividualAcknowledgment(batchMessageId.toMessageIdImpl(),
-                        batchMessageId,
+            MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+            if (MessageIdAdvUtils.isBatch(messageIdAdv)) {
+                addIndividualAcknowledgment(MessageIdAdvUtils.discardBatch(messageIdAdv),
+                        messageIdAdv,
                         this::doIndividualAckAsync,
                         this::doIndividualBatchAckAsync);
-            } else if (messageId instanceof MessageIdImpl) {
-                addIndividualAcknowledgment((MessageIdImpl) messageId,
+            } else {
+                addIndividualAcknowledgment(messageIdAdv,
                         null,
                         this::doIndividualAckAsync,
                         this::doIndividualBatchAckAsync);
-            } else {
-                throw new IllegalStateException("Unsupported message id type in addListAcknowledgement: "
-                        + messageId.getClass().getCanonicalName());
             }
         }
     }
 
     @Override
-    public CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType,
+    public CompletableFuture addAcknowledgment(MessageId msgId, AckType ackType,
                                                      Map properties) {
-        if (msgId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) msgId;
-            return addAcknowledgment(batchMessageId.toMessageIdImpl(), ackType, properties, batchMessageId);
+        MessageIdAdv msgIdAdv = (MessageIdAdv) msgId;
+        if (MessageIdAdvUtils.isBatch(msgIdAdv)) {
+            return addAcknowledgment(MessageIdAdvUtils.discardBatch(msgId), ackType, properties, msgIdAdv);
         } else {
-            return addAcknowledgment(msgId, ackType, properties, null);
+            return addAcknowledgment(msgIdAdv, ackType, properties, null);
         }
     }
 
     private CompletableFuture addIndividualAcknowledgment(
-            MessageIdImpl msgId,
-            @Nullable BatchMessageIdImpl batchMessageId,
-            Function> individualAckFunction,
-            Function> batchAckFunction) {
+            MessageIdAdv msgId,
+            @Nullable MessageIdAdv batchMessageId,
+            Function> individualAckFunction,
+            Function> batchAckFunction) {
         if (batchMessageId != null) {
             consumer.onAcknowledge(batchMessageId, null);
         } else {
             consumer.onAcknowledge(msgId, null);
         }
-        if (batchMessageId == null || batchMessageId.ackIndividual()) {
+        if (batchMessageId == null || MessageIdAdvUtils.acknowledge(batchMessageId, true)) {
             consumer.getStats().incrementNumAcksSent((batchMessageId != null) ? batchMessageId.getBatchSize() : 1);
             consumer.getUnAckedMessageTracker().remove(msgId);
             if (consumer.getPossibleSendToDeadLetterTopicMessages() != null) {
@@ -215,10 +212,10 @@ private CompletableFuture addIndividualAcknowledgment(
         }
     }
 
-    private CompletableFuture addAcknowledgment(MessageIdImpl msgId,
+    private CompletableFuture addAcknowledgment(MessageIdAdv msgId,
                                                       AckType ackType,
                                                       Map properties,
-                                                      @Nullable BatchMessageIdImpl batchMessageId) {
+                                                      @Nullable MessageIdAdv batchMessageId) {
         switch (ackType) {
             case Individual:
                 return addIndividualAcknowledgment(msgId,
@@ -231,15 +228,12 @@ private CompletableFuture addAcknowledgment(MessageIdImpl msgId,
                 } else {
                     consumer.onAcknowledgeCumulative(msgId, null);
                 }
-                if (batchMessageId == null || batchMessageId.ackCumulative()) {
+                if (batchMessageId == null || MessageIdAdvUtils.acknowledge(batchMessageId, false)) {
                     return doCumulativeAck(msgId, properties, null);
                 } else if (batchIndexAckEnabled) {
                     return doCumulativeBatchIndexAck(batchMessageId, properties);
                 } else {
-                    if (!batchMessageId.getAcker().isPrevBatchCumulativelyAcked()) {
-                        doCumulativeAck(batchMessageId.prevBatchMessageId(), properties, null);
-                        batchMessageId.getAcker().setPrevBatchCumulativelyAcked(true);
-                    }
+                    doCumulativeAck(MessageIdAdvUtils.prevMessageId(batchMessageId), properties, null);
                     return CompletableFuture.completedFuture(null);
                 }
             default:
@@ -247,7 +241,7 @@ private CompletableFuture addAcknowledgment(MessageIdImpl msgId,
         }
     }
 
-    private CompletableFuture doIndividualAck(MessageIdImpl messageId, Map properties) {
+    private CompletableFuture doIndividualAck(MessageIdAdv messageId, Map properties) {
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
             // We cannot group acks if the delay is 0 or when there are properties attached to it. Fortunately that's an
             // uncommon condition since it's only used for the compaction subscription.
@@ -267,13 +261,13 @@ private CompletableFuture doIndividualAck(MessageIdImpl messageId, Map doIndividualAckAsync(MessageIdImpl messageId) {
+    private CompletableFuture doIndividualAckAsync(MessageIdAdv messageId) {
         pendingIndividualAcks.add(messageId);
         pendingIndividualBatchIndexAcks.remove(messageId);
         return CompletableFuture.completedFuture(null);
     }
 
-    private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMessageId,
+    private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId,
                                                          Map properties) {
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
             return doImmediateBatchIndexAck(batchMessageId, batchMessageId.getBatchIndex(),
@@ -283,7 +277,7 @@ private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMes
         }
     }
 
-    private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMessageId) {
+    private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId) {
         Optional readLock = acquireReadLock();
         try {
             doIndividualBatchAckAsync(batchMessageId);
@@ -296,7 +290,7 @@ private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMes
         }
     }
 
-    private CompletableFuture doCumulativeAck(MessageIdImpl messageId, Map properties,
+    private CompletableFuture doCumulativeAck(MessageIdAdv messageId, Map properties,
                                                     BitSetRecyclable bitSet) {
         consumer.getStats().incrementNumAcksSent(consumer.getUnAckedMessageTracker().removeMessagesTill(messageId));
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
@@ -314,29 +308,29 @@ private CompletableFuture doCumulativeAck(MessageIdImpl messageId, Map doIndividualBatchAckAsync(BatchMessageIdImpl batchMessageId) {
+    private CompletableFuture doIndividualBatchAckAsync(MessageIdAdv msgId) {
         ConcurrentBitSetRecyclable bitSet = pendingIndividualBatchIndexAcks.computeIfAbsent(
-                batchMessageId.toMessageIdImpl(), __ -> {
-                    ConcurrentBitSetRecyclable value;
-                    if (batchMessageId.getAcker() != null
-                            && !(batchMessageId.getAcker() instanceof BatchMessageAckerDisabled)) {
-                        value = ConcurrentBitSetRecyclable.create(batchMessageId.getAcker().getBitSet());
+                MessageIdAdvUtils.discardBatch(msgId), __ -> {
+                    final BitSet ackSet = msgId.getAckSet();
+                    final ConcurrentBitSetRecyclable value;
+                    if (ackSet != null && !ackSet.isEmpty()) {
+                        value = ConcurrentBitSetRecyclable.create(ackSet);
                     } else {
                         value = ConcurrentBitSetRecyclable.create();
-                        value.set(0, batchMessageId.getOriginalBatchSize());
+                        value.set(0, msgId.getBatchSize());
                     }
                     return value;
                 });
-        bitSet.clear(batchMessageId.getBatchIndex());
+        bitSet.clear(msgId.getBatchIndex());
         return CompletableFuture.completedFuture(null);
     }
 
-    private void doCumulativeAckAsync(MessageIdImpl msgId, BitSetRecyclable bitSet) {
+    private void doCumulativeAckAsync(MessageIdAdv msgId, BitSetRecyclable bitSet) {
         // Handle concurrent updates from different threads
         lastCumulativeAck.update(msgId, bitSet);
     }
 
-    private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl batchMessageId,
+    private CompletableFuture doCumulativeBatchIndexAck(MessageIdAdv batchMessageId,
                                                               Map properties) {
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
             return doImmediateBatchIndexAck(batchMessageId, batchMessageId.getBatchIndex(),
@@ -349,7 +343,7 @@ private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl bat
         }
     }
 
-    private CompletableFuture doImmediateAck(MessageIdImpl msgId, AckType ackType, Map properties,
+    private CompletableFuture doImmediateAck(MessageIdAdv msgId, AckType ackType, Map properties,
                                                    BitSetRecyclable bitSet) {
         ClientCnx cnx = consumer.getClientCnx();
 
@@ -360,7 +354,7 @@ private CompletableFuture doImmediateAck(MessageIdImpl msgId, AckType ackT
         return newImmediateAckAndFlush(consumer.consumerId, msgId, bitSet, ackType, properties, cnx);
     }
 
-    private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgId, int batchIndex, int batchSize,
+    private CompletableFuture doImmediateBatchIndexAck(MessageIdAdv msgId, int batchIndex, int batchSize,
                                                              AckType ackType, Map properties) {
         ClientCnx cnx = consumer.getClientCnx();
 
@@ -369,8 +363,8 @@ private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgI
                     .ConnectException("Consumer connect fail! consumer state:" + consumer.getState()));
         }
         BitSetRecyclable bitSet;
-        if (msgId.getAcker() != null && !(msgId.getAcker() instanceof BatchMessageAckerDisabled)) {
-            bitSet = BitSetRecyclable.valueOf(msgId.getAcker().getBitSet().toLongArray());
+        if (msgId.getAckSet() != null) {
+            bitSet = BitSetRecyclable.valueOf(msgId.getAckSet().toLongArray());
         } else {
             bitSet = BitSetRecyclable.create();
             bitSet.set(0, batchSize);
@@ -382,7 +376,7 @@ private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgI
         }
 
         CompletableFuture completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId,
-                msgId.ledgerId, msgId.entryId, bitSet, ackType, properties, true, null, null);
+                msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null);
         bitSet.recycle();
         return completableFuture;
     }
@@ -414,7 +408,7 @@ private void flushAsync(ClientCnx cnx) {
         boolean shouldFlush = false;
         if (lastCumulativeAckToFlush != null) {
             shouldFlush = true;
-            final MessageIdImpl messageId = lastCumulativeAckToFlush.getMessageId();
+            final MessageIdAdv messageId = lastCumulativeAckToFlush.getMessageId();
             newMessageAckCommandAndWrite(cnx, consumer.consumerId, messageId.getLedgerId(), messageId.getEntryId(),
                     lastCumulativeAckToFlush.getBitSetRecyclable(), AckType.Cumulative,
                     Collections.emptyMap(), false,
@@ -429,7 +423,7 @@ private void flushAsync(ClientCnx cnx) {
             if (Commands.peerSupportsMultiMessageAcknowledgment(cnx.getRemoteEndpointProtocolVersion())) {
                 // We can send 1 single protobuf command with all individual acks
                 while (true) {
-                    MessageIdImpl msgId = pendingIndividualAcks.pollFirst();
+                    MessageIdAdv msgId = pendingIndividualAcks.pollFirst();
                     if (msgId == null) {
                         break;
                     }
@@ -452,7 +446,7 @@ private void flushAsync(ClientCnx cnx) {
             } else {
                 // When talking to older brokers, send the acknowledgements individually
                 while (true) {
-                    MessageIdImpl msgId = pendingIndividualAcks.pollFirst();
+                    MessageIdAdv msgId = pendingIndividualAcks.pollFirst();
                     if (msgId == null) {
                         break;
                     }
@@ -465,12 +459,13 @@ private void flushAsync(ClientCnx cnx) {
         }
 
         if (!pendingIndividualBatchIndexAcks.isEmpty()) {
-            Iterator> iterator =
+            Iterator> iterator =
                     pendingIndividualBatchIndexAcks.entrySet().iterator();
 
             while (iterator.hasNext()) {
-                Map.Entry entry = iterator.next();
-                entriesToAck.add(Triple.of(entry.getKey().ledgerId, entry.getKey().entryId, entry.getValue()));
+                Map.Entry entry = iterator.next();
+                entriesToAck.add(Triple.of(
+                        entry.getKey().getLedgerId(), entry.getKey().getEntryId(), entry.getValue()));
                 iterator.remove();
             }
         }
@@ -509,7 +504,7 @@ public void close() {
         }
     }
 
-    private CompletableFuture newImmediateAckAndFlush(long consumerId, MessageIdImpl msgId,
+    private CompletableFuture newImmediateAckAndFlush(long consumerId, MessageIdAdv msgId,
                                                             BitSetRecyclable bitSet, AckType ackType,
                                                             Map map, ClientCnx cnx) {
         MessageIdImpl[] chunkMsgIds = this.consumer.unAckedChunkedMessageIdSequenceMap.remove(msgId);
@@ -535,7 +530,7 @@ private CompletableFuture newImmediateAckAndFlush(long consumerId, Message
                 completableFuture = CompletableFuture.completedFuture(null);
             }
         } else {
-            completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.ledgerId, msgId.getEntryId(),
+            completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.getLedgerId(), msgId.getEntryId(),
                     bitSet, ackType, map, true, null, null);
         }
         return completableFuture;
@@ -621,13 +616,13 @@ protected LastCumulativeAck initialValue() {
                     return new LastCumulativeAck();
                 }
             };
-    public static final MessageIdImpl DEFAULT_MESSAGE_ID = (MessageIdImpl) MessageIdImpl.earliest;
+    public static final MessageIdAdv DEFAULT_MESSAGE_ID = (MessageIdAdv) MessageId.earliest;
 
-    private volatile MessageIdImpl messageId = DEFAULT_MESSAGE_ID;
+    private volatile MessageIdAdv messageId = DEFAULT_MESSAGE_ID;
     private BitSetRecyclable bitSetRecyclable = null;
     private boolean flushRequired = false;
 
-    public synchronized void update(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) {
+    public synchronized void update(final MessageIdAdv messageId, final BitSetRecyclable bitSetRecyclable) {
         if (compareTo(messageId) < 0) {
             if (this.bitSetRecyclable != null && this.bitSetRecyclable != bitSetRecyclable) {
                 this.bitSetRecyclable.recycle();
@@ -662,25 +657,22 @@ public synchronized void reset() {
         flushRequired = false;
     }
 
-    public synchronized int compareTo(MessageId messageId) {
-        if (this.messageId instanceof BatchMessageIdImpl && (!(messageId instanceof BatchMessageIdImpl))) {
-            final BatchMessageIdImpl lhs = (BatchMessageIdImpl) this.messageId;
-            final MessageIdImpl rhs = (MessageIdImpl) messageId;
-            return MessageIdImpl.messageIdCompare(
-                    lhs.getLedgerId(), lhs.getEntryId(), lhs.getPartitionIndex(), lhs.getBatchIndex(),
-                    rhs.getLedgerId(), rhs.getEntryId(), rhs.getPartitionIndex(), Integer.MAX_VALUE);
-        } else if (messageId instanceof BatchMessageIdImpl && (!(this.messageId instanceof BatchMessageIdImpl))){
-            final MessageIdImpl lhs = this.messageId;
-            final BatchMessageIdImpl rhs = (BatchMessageIdImpl) messageId;
-            return MessageIdImpl.messageIdCompare(
-                    lhs.getLedgerId(), lhs.getEntryId(), lhs.getPartitionIndex(), Integer.MAX_VALUE,
-                    rhs.getLedgerId(), rhs.getEntryId(), rhs.getPartitionIndex(), rhs.getBatchIndex());
-        } else {
-            return this.messageId.compareTo(messageId);
+    public synchronized int compareTo(MessageIdAdv messageId) {
+        int result = Long.compare(this.messageId.getLedgerId(), messageId.getLedgerId());
+        if (result != 0) {
+            return result;
+        }
+        result = Long.compare(this.messageId.getEntryId(), messageId.getEntryId());
+        if (result != 0) {
+            return result;
         }
+        return Integer.compare(
+                (this.messageId.getBatchIndex() >= 0) ? this.messageId.getBatchIndex() : Integer.MAX_VALUE,
+                (messageId.getBatchIndex() >= 0) ? messageId.getBatchIndex() : Integer.MAX_VALUE
+        );
     }
 
-    private synchronized void set(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) {
+    private synchronized void set(final MessageIdAdv messageId, final BitSetRecyclable bitSetRecyclable) {
         this.messageId = messageId;
         this.bitSetRecyclable = bitSetRecyclable;
     }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java
index 06f79024c4f24..1c4230470dbc2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java
@@ -22,6 +22,8 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
+import org.apache.pulsar.client.api.TopicMessageId;
 
 @Data
 @NoArgsConstructor
@@ -67,18 +69,12 @@ private ResetCursorData(String position) {
     }
 
     public ResetCursorData(MessageId messageId) {
-        if (messageId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-            this.ledgerId = batchMessageId.getLedgerId();
-            this.entryId = batchMessageId.getEntryId();
-            this.batchIndex = batchMessageId.getBatchIndex();
-            this.partitionIndex = batchMessageId.partitionIndex;
-        } else if (messageId instanceof MessageIdImpl) {
-            MessageIdImpl messageIdImpl = (MessageIdImpl) messageId;
-            this.ledgerId = messageIdImpl.getLedgerId();
-            this.entryId = messageIdImpl.getEntryId();
-            this.partitionIndex = messageIdImpl.partitionIndex;
-        }  else if (messageId instanceof TopicMessageIdImpl) {
+        MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+        this.ledgerId = messageIdAdv.getLedgerId();
+        this.entryId = messageIdAdv.getEntryId();
+        this.batchIndex = messageIdAdv.getBatchIndex();
+        this.partitionIndex = messageIdAdv.getPartitionIndex();
+        if (messageId instanceof TopicMessageId) {
             throw new IllegalArgumentException("Not supported operation on partitioned-topic");
         }
     }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
index 941f18cf65a2c..189dc1c608379 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
@@ -21,16 +21,12 @@
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.TopicMessageId;
 
-public class TopicMessageIdImpl implements TopicMessageId {
+public class TopicMessageIdImpl extends TopicMessageId.Impl {
 
-    /** This topicPartitionName is get from ConsumerImpl, it contains partition part. */
-    private final String topicPartitionName;
     private final String topicName;
-    private final MessageId messageId;
 
     public TopicMessageIdImpl(String topicPartitionName, String topicName, MessageId messageId) {
-        this.messageId = messageId;
-        this.topicPartitionName = topicPartitionName;
+        super(topicPartitionName, messageId);
         this.topicName = topicName;
     }
 
@@ -49,40 +45,21 @@ public String getTopicName() {
      */
     @Deprecated
     public String getTopicPartitionName() {
-        return this.topicPartitionName;
+        return getOwnerTopic();
     }
 
+    @Deprecated
     public MessageId getInnerMessageId() {
-        return messageId;
-    }
-
-    @Override
-    public String toString() {
-        return messageId.toString();
-    }
-
-    @Override
-    public byte[] toByteArray() {
-        return messageId.toByteArray();
-    }
-
-    @Override
-    public int hashCode() {
-        return messageId.hashCode();
+        return new MessageIdImpl(getLedgerId(), getEntryId(), getPartitionIndex());
     }
 
     @Override
     public boolean equals(Object obj) {
-        return messageId.equals(obj);
+        return super.equals(obj);
     }
 
     @Override
-    public int compareTo(MessageId o) {
-        return messageId.compareTo(o);
-    }
-
-    @Override
-    public String getOwnerTopic() {
-        return topicPartitionName;
+    public int hashCode() {
+        return super.hashCode();
     }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
index c3fcb0a16a383..d24ecbd6aa917 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
@@ -70,7 +70,7 @@ public MessageId getMessageId() {
 
     @Deprecated
     public MessageId getInnerMessageId() {
-        return MessageIdImpl.convertToMessageIdImpl(messageId);
+        return messageId.getInnerMessageId();
     }
 
     @Override
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
index 42eb197d632d3..ae874b4da6d6b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
@@ -174,7 +174,8 @@ private void triggerZeroQueueSizeListener(final Message message) {
                 }
                 waitingOnListenerForZeroQueueSize = true;
                 trackMessage(message);
-                unAckedMessageTracker.add(normalizeMessageId(message.getMessageId()), message.getRedeliveryCount());
+                unAckedMessageTracker.add(
+                        MessageIdAdvUtils.discardBatch(message.getMessageId()), message.getRedeliveryCount());
                 listener.received(ZeroQueueConsumerImpl.this, beforeConsume(message));
             } catch (Throwable t) {
                 log.error("[{}][{}] Message listener error in processing unqueued message: {}", topic, subscription,
diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml
index e5f8babe841b8..92ec9e934ee1e 100644
--- a/pulsar-client/src/main/resources/findbugsExclude.xml
+++ b/pulsar-client/src/main/resources/findbugsExclude.xml
@@ -1007,4 +1007,9 @@
         
         
     
+    
+        
+        
+        
+    
 
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java
index ddca6951e49e1..0418a54c772cc 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java
@@ -38,6 +38,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
 import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
 import org.apache.pulsar.client.util.TimedCompletableFuture;
@@ -61,7 +62,7 @@ public void setup() throws NoSuchFieldException, IllegalAccessException {
         eventLoopGroup = new NioEventLoopGroup(1);
         consumer = mock(ConsumerImpl.class);
         consumer.unAckedChunkedMessageIdSequenceMap =
-                ConcurrentOpenHashMap.newBuilder().build();
+                ConcurrentOpenHashMap.newBuilder().build();
         cnx = spy(new ClientCnxTest(new ClientConfigurationData(), eventLoopGroup));
         PulsarClientImpl client = mock(PulsarClientImpl.class);
         doReturn(client).when(consumer).getClient();
@@ -391,21 +392,21 @@ public void testBatchAckTrackerMultiAck(boolean isNeedReceipt) throws Exception
     public void testDoIndividualBatchAckAsync() throws Exception{
         ConsumerConfigurationData conf = new ConsumerConfigurationData<>();
         AcknowledgmentsGroupingTracker tracker = new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup);
-        MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, BatchMessageAckerDisabled.INSTANCE);
+        MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, null);
         BitSet bitSet = new BitSet(20);
         for(int i = 0; i < 20; i ++) {
             bitSet.set(i, true);
         }
-        MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, BatchMessageAcker.newAcker(bitSet));
+        MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, bitSet);
         Method doIndividualBatchAckAsync = PersistentAcknowledgmentsGroupingTracker.class
-                .getDeclaredMethod("doIndividualBatchAckAsync", BatchMessageIdImpl.class);
+                .getDeclaredMethod("doIndividualBatchAckAsync", MessageIdAdv.class);
         doIndividualBatchAckAsync.setAccessible(true);
         doIndividualBatchAckAsync.invoke(tracker, messageId1);
         doIndividualBatchAckAsync.invoke(tracker, messageId2);
         Field pendingIndividualBatchIndexAcks = PersistentAcknowledgmentsGroupingTracker.class.getDeclaredField("pendingIndividualBatchIndexAcks");
         pendingIndividualBatchIndexAcks.setAccessible(true);
-        ConcurrentHashMap batchIndexAcks =
-                (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker);
+        ConcurrentHashMap batchIndexAcks =
+                (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker);
         MessageIdImpl position1 = new MessageIdImpl(5, 1, 0);
         MessageIdImpl position2 = new MessageIdImpl(3, 2, 0);
         assertTrue(batchIndexAcks.containsKey(position1));
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java
deleted file mode 100644
index 1b3795d878cd8..0000000000000
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-import org.testng.annotations.Test;
-
-public class BatchMessageAckerDisabledTest {
-
-    @Test
-    public void testAckIndividual() {
-        for (int i = 0; i < 10; i++) {
-            assertTrue(BatchMessageAckerDisabled.INSTANCE.ackIndividual(i));
-        }
-    }
-
-    @Test
-    public void testAckCumulative() {
-        for (int i = 0; i < 10; i++) {
-            assertTrue(BatchMessageAckerDisabled.INSTANCE.ackCumulative(i));
-        }
-    }
-
-    @Test
-    public void testGetOutstandingAcks() {
-        assertEquals(0, BatchMessageAckerDisabled.INSTANCE.getOutstandingAcks());
-    }
-
-}
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java
deleted file mode 100644
index d31fd18cba971..0000000000000
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.BitSet;
-
-public class BatchMessageAckerTest {
-
-    private static final int BATCH_SIZE = 10;
-
-    private BatchMessageAcker acker;
-
-    @BeforeMethod
-    public void setup() {
-        acker = BatchMessageAcker.newAcker(10);
-    }
-
-    @Test
-    public void testAckers() {
-        assertEquals(BATCH_SIZE, acker.getOutstandingAcks());
-        assertEquals(BATCH_SIZE, acker.getBatchSize());
-
-        assertFalse(acker.ackIndividual(4));
-        for (int i = 0; i < BATCH_SIZE; i++) {
-            if (4 == i) {
-                assertFalse(acker.getBitSet().get(i));
-            } else {
-                assertTrue(acker.getBitSet().get(i));
-            }
-        }
-
-        assertFalse(acker.ackCumulative(6));
-        for (int i = 0; i < BATCH_SIZE; i++) {
-            if (i <= 6) {
-                assertFalse(acker.getBitSet().get(i));
-            } else {
-                assertTrue(acker.getBitSet().get(i));
-            }
-        }
-
-        for (int i = BATCH_SIZE - 1; i >= 8; i--) {
-            assertFalse(acker.ackIndividual(i));
-            assertFalse(acker.getBitSet().get(i));
-        }
-
-        assertTrue(acker.ackIndividual(7));
-        assertEquals(0, acker.getOutstandingAcks());
-    }
-
-    @Test
-    public void testBitSetAcker() {
-        BitSet bitSet = BitSet.valueOf(acker.getBitSet().toLongArray());
-        BatchMessageAcker bitSetAcker = BatchMessageAcker.newAcker(bitSet);
-
-        Assert.assertEquals(acker.getBitSet(), bitSetAcker.getBitSet());
-        Assert.assertEquals(acker.getOutstandingAcks(), bitSetAcker.getOutstandingAcks());
-    }
-
-}
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
index 6bf9cd943483f..10d805cdc4db3 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
@@ -20,13 +20,8 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectWriter;
 import java.io.IOException;
 import java.util.Collections;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.testng.annotations.Test;
 
 public class BatchMessageIdImplTest {
@@ -123,36 +118,10 @@ public void hashCodeUnbatchedTest() {
         assertEquals(batchMsgId2.hashCode(), msgId2.hashCode());
     }
 
-    @Test
-    public void deserializationTest() {
-        // initialize BitSet with null
-        BatchMessageAcker ackerDisabled = new BatchMessageAcker(null, 0);
-        BatchMessageIdImpl batchMsgId = new BatchMessageIdImpl(0, 0, 0, 0, 0, ackerDisabled);
-
-        ObjectWriter writer = ObjectMapperFactory.create().writerWithDefaultPrettyPrinter();
-
-        try {
-            writer.writeValueAsString(batchMsgId);
-            fail("Shouldn't be deserialized");
-        } catch (JsonProcessingException e) {
-            // expected
-            assertTrue(e.getCause() instanceof NullPointerException);
-        }
-
-        // use the default BatchMessageAckerDisabled
-        BatchMessageIdImpl batchMsgIdToDeserialize = new BatchMessageIdImpl(0, 0, 0, 0);
-
-        try {
-            writer.writeValueAsString(batchMsgIdToDeserialize);
-        } catch (JsonProcessingException e) {
-            fail("Should be successful");
-        }
-    }
-
     @Test
     public void serializeAndDeserializeTest() throws IOException {
         BatchMessageIdImpl batchMessageId = new BatchMessageIdImpl(1, 1, 0,
-            1, 10, BatchMessageAcker.newAcker(10));
+            1, 10, BatchMessageIdImpl.newAckSet(10));
         byte[] serialized = batchMessageId.toByteArray();
         BatchMessageIdImpl deserialized = (BatchMessageIdImpl) MessageIdImpl.fromByteArray(serialized);
         assertEquals(deserialized.getBatchSize(), batchMessageId.getBatchSize());
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java
index 7f029635241de..4173d6439b931 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java
@@ -43,8 +43,7 @@ public void testProtobufSerialization2() throws Exception {
 
     @Test
     public void testBatchSizeNotSet() throws Exception {
-        MessageId id = new BatchMessageIdImpl(1L, 2L, 3, 4, -1,
-                BatchMessageAckerDisabled.INSTANCE);
+        MessageId id = new BatchMessageIdImpl(1L, 2L, 3, 4, -1, null);
         byte[] serialized = id.toByteArray();
         assertEquals(MessageId.fromByteArray(serialized), id);
         assertEquals(MessageId.fromByteArrayWithTopic(serialized, "my-topic"), id);
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index 28cce0fe62209..7df173da0f195 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -48,6 +48,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic;
@@ -325,7 +326,7 @@ public static String getFullyQualifiedInstanceId(String tenant, String namespace
     }
 
     public static final long getSequenceId(MessageId messageId) {
-        MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(messageId);
+        MessageIdAdv msgId = (MessageIdAdv) messageId;
         long ledgerId = msgId.getLedgerId();
         long entryId = msgId.getEntryId();
 
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index 10efc91ccdaad..ff0bfd391e80e 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -26,6 +26,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -389,6 +390,23 @@ static class BatchMessageSequenceRef {
         int batchIdx;
     }
 
+    private static Method getMethodOfMessageId(MessageId messageId, String name) throws NoSuchMethodException {
+        Class clazz = messageId.getClass();
+        NoSuchMethodException firstException = null;
+        while (clazz != null) {
+            try {
+                return clazz.getDeclaredMethod(name);
+            } catch (NoSuchMethodException e) {
+                if (firstException == null) {
+                    firstException = e;
+                }
+                clazz = clazz.getSuperclass();
+            }
+        }
+        assert firstException != null;
+        throw firstException;
+    }
+
     @VisibleForTesting
     static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId messageId) {
         long ledgerId;
@@ -396,23 +414,17 @@ static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId me
         int batchIdx;
         try {
             try {
-                messageId = (MessageId) messageId.getClass().getDeclaredMethod("getInnerMessageId").invoke(messageId);
-            } catch (NoSuchMethodException noSuchMethodException) {
-                // not a TopicMessageIdImpl
-            }
-
-            try {
-                batchIdx = (int) messageId.getClass().getDeclaredMethod("getBatchIndex").invoke(messageId);
+                batchIdx = (int) getMethodOfMessageId(messageId, "getBatchIndex").invoke(messageId);
+                if (batchIdx < 0) {
+                    return null;
+                }
             } catch (NoSuchMethodException noSuchMethodException) {
                 // not a BatchMessageIdImpl, returning null to use the standard sequenceId
                 return null;
             }
 
-            // if getBatchIndex exists it means messageId is a 'BatchMessageIdImpl' instance.
-            final Class messageIdImplClass = messageId.getClass().getSuperclass();
-
-            ledgerId = (long) messageIdImplClass.getDeclaredMethod("getLedgerId").invoke(messageId);
-            entryId = (long) messageIdImplClass.getDeclaredMethod("getEntryId").invoke(messageId);
+            ledgerId = (long) getMethodOfMessageId(messageId, "getLedgerId").invoke(messageId);
+            entryId = (long) getMethodOfMessageId(messageId, "getEntryId").invoke(messageId);
         } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException ex) {
             log.error("Unexpected error while retrieving sequenceId, messageId class: {}, error: {}",
                     messageId.getClass().getName(), ex.getMessage(), ex);

From d0916754ddeb1a4315e4429601941fdc6a210f30 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 4 Apr 2023 23:09:43 +0300
Subject: [PATCH 135/404] [fix][build] Suppress Guava CVE-2020-8908 in OWASP
 dependency check (#20005)

---
 src/owasp-dependency-check-suppressions.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml
index 2f73564649445..84ed2a3332cd4 100644
--- a/src/owasp-dependency-check-suppressions.xml
+++ b/src/owasp-dependency-check-suppressions.xml
@@ -463,5 +463,12 @@
         CVE-2020-17516
         CVE-2021-44521
     
+    
+        
+        CVE-2020-8908
+    
 
 

From 0d1fe1821e030c0c0b53a7d47a2bf89151783eb4 Mon Sep 17 00:00:00 2001
From: Masahiro Sakamoto 
Date: Wed, 5 Apr 2023 15:45:59 +0900
Subject: [PATCH 136/404] [refactor][broker] Suppress error logging when
 message expiration fails (#19778)

---
 .../admin/impl/PersistentTopicsBase.java      | 85 +++++++++++++------
 1 file changed, 59 insertions(+), 26 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 035e32542ed71..7347d6dbf20a2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -2098,11 +2098,15 @@ protected void internalExpireMessagesForAllSubscriptions(AsyncResponse asyncResp
 
                             FutureUtil.waitForAll(futures).handle((result, exception) -> {
                                 if (exception != null) {
-                                    Throwable t = exception.getCause();
-                                    log.error("[{}] Failed to expire messages up to {} on {}",
-                                            clientAppId(), expireTimeInSeconds,
-                                            topicName, t);
-                                    asyncResponse.resume(new RestException(t));
+                                    Throwable t = FutureUtil.unwrapCompletionException(exception);
+                                    if (t instanceof PulsarAdminException) {
+                                        log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(),
+                                                expireTimeInSeconds, topicName, t.toString());
+                                    } else {
+                                        log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(),
+                                                expireTimeInSeconds, topicName, t);
+                                    }
+                                    resumeAsyncResponseExceptionally(asyncResponse, t);
                                     return null;
                                 }
                                 asyncResponse.resume(Response.noContent().build());
@@ -2169,9 +2173,14 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy
                     FutureUtil.waitForAll(futures).handle((result, exception) -> {
                         if (exception != null) {
                             Throwable throwable = FutureUtil.unwrapCompletionException(exception);
-                            log.error("[{}] Failed to expire messages for all subscription up to {} on {}",
-                                    clientAppId(), expireTimeInSeconds, topicName, throwable);
-                            asyncResponse.resume(new RestException(throwable));
+                            if (throwable instanceof RestException) {
+                                log.warn("[{}] Failed to expire messages for all subscription up to {} on {}: {}",
+                                        clientAppId(), expireTimeInSeconds, topicName, throwable.toString());
+                            } else {
+                                log.error("[{}] Failed to expire messages for all subscription up to {} on {}",
+                                        clientAppId(), expireTimeInSeconds, topicName, throwable);
+                            }
+                            resumeAsyncResponseExceptionally(asyncResponse, throwable);
                             return null;
                         }
                         asyncResponse.resume(Response.noContent().build());
@@ -3927,17 +3936,24 @@ protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, St
 
                                                 FutureUtil.waitForAll(futures).handle((result, exception) -> {
                                                     if (exception != null) {
-                                                        Throwable t = exception.getCause();
+                                                        Throwable t = FutureUtil.unwrapCompletionException(exception);
                                                         if (t instanceof NotFoundException) {
                                                             asyncResponse.resume(new RestException(Status.NOT_FOUND,
                                                                     getSubNotFoundErrorMessage(topicName.toString(),
                                                                             subName)));
                                                             return null;
                                                         } else {
-                                                            log.error("[{}] Failed to expire messages up "
-                                                                            + "to {} on {}", clientAppId(),
-                                                                    expireTimeInSeconds, topicName, t);
-                                                            asyncResponse.resume(new RestException(t));
+                                                            if (t instanceof PulsarAdminException) {
+                                                                log.warn("[{}] Failed to expire messages up "
+                                                                        + "to {} on {}: {}", clientAppId(),
+                                                                                expireTimeInSeconds, topicName,
+                                                                                        t.toString());
+                                                            } else {
+                                                                log.error("[{}] Failed to expire messages up "
+                                                                        + "to {} on {}", clientAppId(),
+                                                                                expireTimeInSeconds, topicName, t);
+                                                            }
+                                                            resumeAsyncResponseExceptionally(asyncResponse, t);
                                                             return null;
                                                         }
                                                     }
@@ -3955,12 +3971,18 @@ protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, St
                                 }))
 
         ).exceptionally(ex -> {
+            Throwable cause = FutureUtil.unwrapCompletionException(ex);
             // If the exception is not redirect exception we need to log it.
-            if (!isRedirectException(ex)) {
-                log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(),
-                        expireTimeInSeconds, topicName, ex);
+            if (!isRedirectException(cause)) {
+                if (cause instanceof RestException) {
+                    log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), expireTimeInSeconds,
+                            topicName, cause.toString());
+                } else {
+                    log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(), expireTimeInSeconds,
+                            topicName, cause);
+                }
             }
-            resumeAsyncResponseExceptionally(asyncResponse, ex);
+            resumeAsyncResponseExceptionally(asyncResponse, cause);
             return null;
         });
     }
@@ -4068,7 +4090,7 @@ protected void internalExpireMessagesByPosition(AsyncResponse asyncResponse, Str
         future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative))
                 .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, subName))
                 .thenCompose(__ -> {
-                    log.info("[{}][{}] received expire messages on subscription {} to position {}", clientAppId(),
+                    log.info("[{}][{}] Received expire messages on subscription {} to position {}", clientAppId(),
                             topicName, subName, messageId);
                     return internalExpireMessagesNonPartitionedTopicByPosition(asyncResponse, subName,
                             messageId, isExcluded, batchIndex);
@@ -4134,16 +4156,22 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit
                                     + topicName + " for subscription " + subName + " due to ongoing"
                                     + " message expiration not finished or invalid message position provided.");
                         }
+                    } catch (RestException exception) {
+                        throw exception;
                     } catch (Exception exception) {
-                        log.error("[{}] Failed to expire messages up to {} on {} with subscription {} {}",
-                                clientAppId(), position, topicName, subName, exception);
                         throw new RestException(exception);
                     }
                     asyncResponse.resume(Response.noContent().build());
                 }).exceptionally(e -> {
-                    log.error("[{}] Failed to expire messages up to {} on {} with subscription {} {}",
-                            clientAppId(), messageId, topicName, subName, e);
-                    asyncResponse.resume(e);
+                    Throwable throwable = FutureUtil.unwrapCompletionException(e);
+                    if (throwable instanceof RestException) {
+                        log.warn("[{}] Failed to expire messages up to {} on {} with subscription {}: {}",
+                                clientAppId(), messageId, topicName, subName, throwable.toString());
+                    } else {
+                        log.error("[{}] Failed to expire messages up to {} on {} with subscription {}", clientAppId(),
+                                messageId, topicName, subName, throwable);
+                    }
+                    resumeAsyncResponseExceptionally(asyncResponse, throwable);
                     return null;
                 });
             } catch (Exception e) {
@@ -4152,9 +4180,14 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit
                 resumeAsyncResponseExceptionally(asyncResponse, e);
             }
         }).exceptionally(ex -> {
-            Throwable cause = ex.getCause();
-            log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", clientAppId(),
-                    topicName, subName, messageId, cause);
+            Throwable cause = FutureUtil.unwrapCompletionException(ex);
+            if (cause instanceof RestException) {
+                log.warn("[{}] Failed to expire messages up to {} on subscription {} to position {}: {}", clientAppId(),
+                        topicName, subName, messageId, cause.toString());
+            } else {
+                log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", clientAppId(),
+                        topicName, subName, messageId, cause);
+            }
             resumeAsyncResponseExceptionally(asyncResponse, cause);
             return null;
         });

From 7813daba863c45f19ab5d8718c961b8aa97137e7 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Wed, 5 Apr 2023 17:41:54 +0800
Subject: [PATCH 137/404] [improve][broker] PIP-192: Filter the transfer dest
 broker (#19958)

---
 .../extensions/ExtensibleLoadManagerImpl.java |  14 +-
 .../filter/AntiAffinityGroupPolicyFilter.java |   6 -
 .../extensions/filter/BrokerFilter.java       |   6 -
 .../filter/BrokerIsolationPoliciesFilter.java |  11 +-
 .../filter/BrokerMaxTopicCountFilter.java     |   6 -
 .../filter/BrokerVersionFilter.java           |   6 -
 .../AntiAffinityGroupPolicyHelper.java        |  35 +---
 .../policies/IsolationPoliciesHelper.java     |   5 +
 .../extensions/scheduler/TransferShedder.java |  92 +++++------
 ...tiAffinityNamespaceGroupExtensionTest.java |  80 +---------
 .../ExtensibleLoadManagerImplTest.java        |  10 --
 .../BrokerIsolationPoliciesFilterTest.java    |  12 +-
 .../scheduler/TransferShedderTest.java        | 151 ++++++++++++++----
 13 files changed, 195 insertions(+), 239 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 3e078d0a5eb20..7e84fa5969a6d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -64,6 +64,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler;
@@ -75,6 +76,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceName;
@@ -119,6 +121,9 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
     @Getter
     private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
+    @Getter
+    private IsolationPoliciesHelper isolationPoliciesHelper;
+
     private LoadDataStore brokerLoadDataStore;
     private LoadDataStore topBundlesLoadDataStore;
 
@@ -185,7 +190,6 @@ public enum Role {
     public ExtensibleLoadManagerImpl() {
         this.brokerFilterPipeline = new ArrayList<>();
         this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter());
-        this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter());
         this.brokerFilterPipeline.add(new BrokerVersionFilter());
         // TODO: Make brokerSelectionStrategy configurable.
         this.brokerSelectionStrategy = new LeastResourceUsageWithWeight();
@@ -236,6 +240,9 @@ public void start() throws PulsarServerException {
         antiAffinityGroupPolicyHelper.listenFailureDomainUpdate();
         this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper);
         this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter);
+        SimpleResourceAllocationPolicies policies = new SimpleResourceAllocationPolicies(pulsar);
+        this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies);
+        this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper));
 
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
@@ -293,8 +300,8 @@ public void start() throws PulsarServerException {
                         MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
 
         this.unloadScheduler = new UnloadScheduler(
-                pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel,
-                unloadCounter, unloadMetrics);
+                pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context,
+                serviceUnitStateChannel, unloadCounter, unloadMetrics);
         this.unloadScheduler.start();
         this.splitScheduler = new SplitScheduler(
                 pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
@@ -307,7 +314,6 @@ public void start() throws PulsarServerException {
     public void initialize(PulsarService pulsar) {
         this.pulsar = pulsar;
         this.conf = pulsar.getConfiguration();
-        this.brokerFilterPipeline.forEach(brokerFilter -> brokerFilter.initialize(pulsar));
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
index 358f985f83e12..462f8f0e3597a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.broker.loadbalance.extensions.filter;
 
 import java.util.Map;
-import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
@@ -50,9 +49,4 @@ public Map filter(
     public String name() {
         return FILTER_NAME;
     }
-
-    @Override
-    public void initialize(PulsarService pulsar) {
-        return;
-    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java
index 30d25f559b11e..d9cbfdc391ed4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.broker.loadbalance.extensions.filter;
 
 import java.util.Map;
-import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
@@ -35,11 +34,6 @@ public interface BrokerFilter {
      */
     String name();
 
-    /**
-     * Initialize this broker filter using the given pulsar service.
-     */
-    void initialize(PulsarService pulsar);
-
     /**
      * Filter out unqualified brokers based on implementation.
      *
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java
index b28c77f76f3eb..eeb0d9d3a3309 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java
@@ -21,12 +21,10 @@
 import java.util.Map;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
-import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 
 
@@ -37,14 +35,13 @@ public class BrokerIsolationPoliciesFilter implements BrokerFilter {
 
     private IsolationPoliciesHelper isolationPoliciesHelper;
 
-    @Override
-    public String name() {
-        return FILTER_NAME;
+    public BrokerIsolationPoliciesFilter(IsolationPoliciesHelper helper) {
+        this.isolationPoliciesHelper = helper;
     }
 
     @Override
-    public void initialize(PulsarService pulsar) {
-        this.isolationPoliciesHelper = new IsolationPoliciesHelper(new SimpleResourceAllocationPolicies(pulsar));
+    public String name() {
+        return FILTER_NAME;
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java
index b98edd3d425e5..0bceae36bb8c2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java
@@ -20,7 +20,6 @@
 
 import java.util.Map;
 import java.util.Optional;
-import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
@@ -36,11 +35,6 @@ public String name() {
         return FILTER_NAME;
     }
 
-    @Override
-    public void initialize(PulsarService pulsar) {
-        // No-op
-    }
-
     @Override
     public Map filter(Map brokers,
                                                 ServiceUnitId serviceUnit,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java
index b7332a5ff10a0..7420fcc211309 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java
@@ -22,7 +22,6 @@
 import java.util.Iterator;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
@@ -148,9 +147,4 @@ public Version getLatestVersionNumber(Map brokerMap)
     public String name() {
         return FILTER_NAME;
     }
-
-    @Override
-    public void initialize(PulsarService pulsar) {
-        // No-op
-    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
index 69e3302bebd50..44360bc77d83f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -20,7 +20,6 @@
 
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
@@ -49,38 +48,10 @@ public void filter(
                 channel.getOwnershipEntrySet(), brokerToFailureDomainMap);
     }
 
-    public boolean canUnload(
-            Map brokers,
-            String bundle,
-            String srcBroker,
-            Optional dstBroker) {
-
+    public boolean hasAntiAffinityGroupPolicy(String bundle) {
         try {
-            var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
-                    pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
-            if (antiAffinityGroupOptional.isEmpty()) {
-                return true;
-            }
-
-            // bundle has anti-affinityGroup
-            if (!pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) {
-                return false;
-            }
-
-            // copy to retain the input brokers
-            Map candidates = new HashMap<>(brokers);
-
-            filter(candidates, bundle);
-
-            candidates.remove(srcBroker);
-
-            // unload case
-            if (dstBroker.isEmpty()) {
-                return !candidates.isEmpty();
-            }
-
-            // transfer case
-            return candidates.containsKey(dstBroker.get());
+            return LoadManagerShared.getNamespaceAntiAffinityGroup(
+                    pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle)).isPresent();
         } catch (MetadataStoreException e) {
             log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e);
             return false;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java
index 4d7a5bf22d661..67dc702cc0c9f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java
@@ -26,6 +26,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
+import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 
 @Slf4j
@@ -65,4 +66,8 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) {
         return brokerCandidateCache;
     }
 
+    public boolean hasIsolationPolicy(NamespaceName namespaceName) {
+        return policies.areIsolationPoliciesPresent(namespaceName);
+    }
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 669219daba273..98e05296d605a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -32,6 +32,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -46,6 +47,7 @@
 import lombok.experimental.Accessors;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
@@ -53,6 +55,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
@@ -60,7 +63,6 @@
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
-import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -96,9 +98,10 @@ public class TransferShedder implements NamespaceUnloadStrategy {
     private static final String CANNOT_UNLOAD_BUNDLE_MSG = "Can't unload bundle:%s.";
     private final LoadStats stats = new LoadStats();
     private PulsarService pulsar;
-    private SimpleResourceAllocationPolicies allocationPolicies;
     private IsolationPoliciesHelper isolationPoliciesHelper;
     private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+    private List brokerFilterPipeline;
+
     private Set decisionCache;
     @Getter
     private UnloadCounter counter;
@@ -109,7 +112,6 @@ public class TransferShedder implements NamespaceUnloadStrategy {
     public TransferShedder(UnloadCounter counter){
         this.pulsar = null;
         this.decisionCache = new HashSet<>();
-        this.allocationPolicies = null;
         this.counter = counter;
         this.isolationPoliciesHelper = null;
         this.antiAffinityGroupPolicyHelper = null;
@@ -117,26 +119,28 @@ public TransferShedder(UnloadCounter counter){
 
     public TransferShedder(PulsarService pulsar,
                            UnloadCounter counter,
+                           List brokerFilterPipeline,
+                           IsolationPoliciesHelper isolationPoliciesHelper,
                            AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper){
         this.pulsar = pulsar;
         this.decisionCache = new HashSet<>();
-        this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
         this.counter = counter;
-        this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
-        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
+        this.isolationPoliciesHelper = isolationPoliciesHelper;
         this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
+        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
+        this.brokerFilterPipeline = brokerFilterPipeline;
     }
 
     @Override
     public void initialize(PulsarService pulsar){
         this.pulsar = pulsar;
         this.decisionCache = new HashSet<>();
-        this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
         var manager = ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get());
         this.counter = manager.getUnloadCounter();
-        this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
-        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
+        this.isolationPoliciesHelper = manager.getIsolationPoliciesHelper();
         this.antiAffinityGroupPolicyHelper = manager.getAntiAffinityGroupPolicyHelper();
+        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
+        this.brokerFilterPipeline = manager.getBrokerFilterPipeline();
     }
 
 
@@ -673,7 +677,7 @@ private boolean isTransferable(LoadManagerContext context,
                                    String bundle,
                                    String srcBroker,
                                    Optional dstBroker) {
-        if (pulsar == null || allocationPolicies == null) {
+        if (pulsar == null) {
             return true;
         }
 
@@ -682,54 +686,50 @@ private boolean isTransferable(LoadManagerContext context,
         NamespaceBundle namespaceBundle =
                 pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespace, bundleRange);
 
-        if (!canTransferWithIsolationPoliciesToBroker(
-                context, availableBrokers, namespaceBundle, srcBroker, dstBroker)) {
+        if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled(context, namespaceBundle)) {
             return false;
         }
 
-        if (antiAffinityGroupPolicyHelper != null
-                && !antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * Check the gave bundle and broker can be transfer or unload with isolation policies applied.
-     *
-     * @param context The load manager context.
-     * @param availableBrokers The available brokers.
-     * @param namespaceBundle The bundle try to unload or transfer.
-     * @param currentBroker The current broker.
-     * @param targetBroker The broker will be transfer to.
-     * @return Can be transfer/unload or not.
-     */
-    private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext context,
-                                                             Map availableBrokers,
-                                                             NamespaceBundle namespaceBundle,
-                                                             String currentBroker,
-                                                             Optional targetBroker) {
-        if (isolationPoliciesHelper == null
-                || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) {
-            return true;
+        Map candidates = new HashMap<>(availableBrokers);
+        for (var filter : brokerFilterPipeline) {
+            try {
+                filter.filter(candidates, namespaceBundle, context);
+            } catch (BrokerFilterException e) {
+                log.error("Failed to filter brokers with filter: {}", filter.getClass().getName(), e);
+                return false;
+            }
         }
 
-        // bundle has isolation policies.
-        if (!context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) {
-            return false;
+        if (dstBroker.isPresent()) {
+            if (!candidates.containsKey(dstBroker.get())) {
+                return false;
+            }
         }
 
-        boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled();
-        Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle);
-
         // Remove the current bundle owner broker.
-        candidates.remove(currentBroker);
+        candidates.remove(srcBroker);
+        boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled();
 
         // Unload: Check if there are any more candidates available for selection.
-        if (targetBroker.isEmpty() || !transfer) {
+        if (dstBroker.isEmpty() || !transfer) {
             return !candidates.isEmpty();
         }
         // Transfer: Check if this broker is among the candidates.
-        return candidates.contains(targetBroker.get());
+        return candidates.containsKey(dstBroker.get());
+    }
+
+    protected boolean isLoadBalancerSheddingBundlesWithPoliciesEnabled(LoadManagerContext context,
+                                                                       NamespaceBundle namespaceBundle) {
+        if (isolationPoliciesHelper != null
+                && isolationPoliciesHelper.hasIsolationPolicy(namespaceBundle.getNamespaceObject())) {
+            return context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled();
+        }
+
+        if (antiAffinityGroupPolicyHelper != null
+                && antiAffinityGroupPolicyHelper.hasAntiAffinityGroupPolicy(namespaceBundle.toString())) {
+            return context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled();
+        }
+
+        return true;
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
index 16b87195b1511..3469dbe5a5499 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -22,8 +22,6 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
 import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -84,83 +82,7 @@ protected void selectBrokerForNamespace(
     }
 
     protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, String bundle) {
-        try {
-            String namespaceBundle = namespace + "/" + bundle;
-            var antiAffinityGroupPolicyHelper =
-                    (AntiAffinityGroupPolicyHelper)
-                            FieldUtils.readDeclaredField(
-                                    primaryLoadManager, "antiAffinityGroupPolicyHelper", true);
-            var brokerRegistry =
-                    (BrokerRegistry)
-                            FieldUtils.readDeclaredField(
-                                    primaryLoadManager, "brokerRegistry", true);
-            var brokers = brokerRegistry
-                    .getAvailableBrokerLookupDataAsync().get(5, TimeUnit.SECONDS);
-            var serviceUnitStateChannel = (ServiceUnitStateChannel)
-                    FieldUtils.readDeclaredField(
-                            primaryLoadManager, "serviceUnitStateChannel", true);
-            var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle)
-                    .get(5, TimeUnit.SECONDS).get();
-            var brokersCopy = new HashMap<>(brokers);
-            brokersCopy.remove(srcBroker);
-            var dstBroker = brokersCopy.entrySet().iterator().next().getKey();
-
-            // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = true
-            conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
-            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    "not-enabled-" + namespace + "/" + bundle,
-                    srcBroker, Optional.of(dstBroker)));
-
-            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    "not-enabled-" + namespace + "/" + bundle,
-                    srcBroker, Optional.empty()));
-
-            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    srcBroker, Optional.of(dstBroker)));
-
-            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    dstBroker, Optional.of(srcBroker)));
-
-            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    srcBroker, Optional.empty()));
-
-            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    dstBroker, Optional.empty()));
-
-            // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = false
-            conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
-            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    "not-enabled-" + namespace + "/" + bundle,
-                    srcBroker, Optional.of(dstBroker)));
-
-            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    "not-enabled-" + namespace + "/" + bundle,
-                    srcBroker, Optional.empty()));
-
-            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    srcBroker, Optional.of(dstBroker)));
-
-            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    dstBroker, Optional.of(srcBroker)));
-
-            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    srcBroker, Optional.empty()));
-
-            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
-                    namespaceBundle,
-                    dstBroker, Optional.empty()));
-
-
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
+        // No-op
     }
 
     protected boolean isLoadManagerUpdatedDomainCache(Object loadManager) throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 4c57e6b93fa3c..502e02e465e64 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -284,11 +284,6 @@ public String name() {
                 return "Mock broker filter";
             }
 
-            @Override
-            public void initialize(PulsarService pulsar) {
-                // No-op
-            }
-
             @Override
             public Map filter(Map brokers,
                                                         ServiceUnitId serviceUnit,
@@ -821,11 +816,6 @@ public String name() {
             return "Mock-broker-filter";
         }
 
-        @Override
-        public void initialize(PulsarService pulsar) {
-            // No-op
-        }
-
     }
 
     private void setPrimaryLoadManager() throws IllegalAccessException {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java
index a079a23bcea04..5bc1e436bae25 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java
@@ -31,7 +31,6 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
-import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
@@ -71,13 +70,13 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker
         var namespace = "my-tenant/my-ns";
         NamespaceName namespaceName = NamespaceName.get(namespace);
 
-        BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter();
         var policies = mock(SimpleResourceAllocationPolicies.class);
 
         // 1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1
         setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 1);
         IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies);
-        FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true);
+
+        BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper);
 
         // a. available-brokers: broker1, broker2, broker3 => result: broker1
         Map result = filter.filter(new HashMap<>(Map.of(
@@ -128,13 +127,14 @@ public void testFilterWithPersistentOrNonPersistentDisabled()
         doReturn(true).when(namespaceBundle).hasNonPersistentTopic();
         doReturn(namespaceName).when(namespaceBundle).getNamespaceObject();
 
-        BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter();
-
         var policies = mock(SimpleResourceAllocationPolicies.class);
         doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName));
         doReturn(true).when(policies).isSharedBroker(any());
         IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies);
-        FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true);
+
+        BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper);
+
+
 
         Map result = filter.filter(new HashMap<>(Map.of(
                 "broker1", getLookupData(),
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 8c8d18e202a00..3279cb4e475d3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -44,9 +44,11 @@
 import com.google.common.collect.BoundType;
 import com.google.common.collect.Range;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
@@ -60,6 +62,7 @@
 import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
@@ -69,6 +72,9 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
@@ -86,6 +92,7 @@
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
 import org.apache.pulsar.common.naming.NamespaceBundles;
 import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.policies.data.LocalPolicies;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
@@ -108,6 +115,7 @@ public class TransferShedderTest {
     ExtensibleLoadManagerImpl loadManager;
     ServiceUnitStateChannel channel;
     ServiceConfiguration conf;
+    IsolationPoliciesHelper isolationPoliciesHelper;
     AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     LocalPoliciesResources localPoliciesResources;
     String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF";
@@ -129,6 +137,7 @@ public void init() throws MetadataStoreException {
         var factory = mock(NamespaceBundleFactory.class);
         namespaceService = mock(NamespaceService.class);
         localPoliciesResources = mock(LocalPoliciesResources.class);
+        isolationPoliciesHelper = mock(IsolationPoliciesHelper.class);
         antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
         doReturn(conf).when(pulsar).getConfiguration();
         doReturn(namespaceService).when(pulsar).getNamespaceService();
@@ -158,8 +167,6 @@ public void init() throws MetadataStoreException {
     public LoadManagerContext setupContext(){
         var ctx = getContext();
 
-
-
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
         topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000));
         topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000));
@@ -389,13 +396,12 @@ public void startTableView() throws LoadDataStoreException {
 
         BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
         doReturn(CompletableFuture.completedFuture(Map.of(
-                "broker1", mock(BrokerLookupData.class),
-                "broker2", mock(BrokerLookupData.class),
-                "broker3", mock(BrokerLookupData.class),
-                "broker4", mock(BrokerLookupData.class),
-                "broker5", mock(BrokerLookupData.class)
-        )))
-                .when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+                "broker1", getMockBrokerLookupData(),
+                "broker2", getMockBrokerLookupData(),
+                "broker3", getMockBrokerLookupData(),
+                "broker4", getMockBrokerLookupData(),
+                "broker5", getMockBrokerLookupData()
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
         doReturn(conf).when(ctx).brokerConfiguration();
         doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore();
         doReturn(topBundleLoadDataStore).when(ctx).topBundleLoadDataStore();
@@ -403,6 +409,14 @@ public void startTableView() throws LoadDataStoreException {
         return ctx;
     }
 
+
+    BrokerLookupData getMockBrokerLookupData() {
+        BrokerLookupData brokerLookupData = mock(BrokerLookupData.class);
+        doReturn(true).when(brokerLookupData).persistentTopicsEnabled();
+        doReturn(true).when(brokerLookupData).nonPersistentTopicsEnabled();
+        return brokerLookupData;
+    }
+
     @Test
     public void testEmptyBrokerLoadData() {
         UnloadCounter counter = new UnloadCounter();
@@ -524,31 +538,33 @@ public void testRecentlyUnloadedBundles() {
     @Test
     public void testGetAvailableBrokersFailed() {
         UnloadCounter counter = new UnloadCounter();
-        AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
-        TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper);
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, null,
+                isolationPoliciesHelper, antiAffinityGroupPolicyHelper);
         var ctx = setupContext();
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
-        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
         assertEquals(counter.getLoadAvg(), 0.0);
         assertEquals(counter.getLoadStd(), 0.0);
     }
 
     @Test(timeOut = 30 * 1000)
-    public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
-        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
-        UnloadCounter counter = new UnloadCounter();
-        TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper));
-        var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
-                spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
-        FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true);
+    public void testBundlesWithIsolationPolicies() {
+        List filters = new ArrayList<>();
+        var allocationPoliciesSpy = mock(SimpleResourceAllocationPolicies.class);
         IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy);
-        FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true);
+        BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper);
+        filters.add(filter);
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, filters,
+                isolationPoliciesHelper, antiAffinityGroupPolicyHelper));
 
         setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
                 Set.of("broker5"), Set.of(), Set.of(), 1);
         var ctx = setupContext();
+        ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
+        doReturn(ctx.brokerConfiguration()).when(pulsar).getConfiguration();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected = new HashSet();
         expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")),
@@ -641,22 +657,25 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies,
 
 
     @Test
-    public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
+    public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException {
+        var filters = new ArrayList();
+        AntiAffinityGroupPolicyFilter filter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper);
+        filters.add(filter);
         var counter = new UnloadCounter();
-        TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper);
-        var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
-                spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
-        doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
-        FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true);
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, filters,
+                isolationPoliciesHelper, antiAffinityGroupPolicyHelper);
 
         LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup");
         doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any());
 
         var ctx = setupContext();
-        var antiAffinityGroupPolicyHelperSpy = (AntiAffinityGroupPolicyHelper)
-                spy(FieldUtils.readDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", true));
-        doReturn(false).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), any(), any(), any());
-        FieldUtils.writeDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", antiAffinityGroupPolicyHelperSpy, true);
+        ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
+
+        doAnswer(invocationOnMock -> {
+            Map brokers = invocationOnMock.getArgument(0);
+            brokers.clear();
+            return brokers;
+        }).when(antiAffinityGroupPolicyHelper).filter(any(), any());
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         assertTrue(res.isEmpty());
@@ -664,8 +683,16 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
 
+        doAnswer(invocationOnMock -> {
+            Map brokers = invocationOnMock.getArgument(0);
+            String bundle = invocationOnMock.getArgument(1, String.class);
 
-        doReturn(true).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), eq(bundleE1), any(), any());
+            if (bundle.equalsIgnoreCase(bundleE1)) {
+                return brokers;
+            }
+            brokers.clear();
+            return brokers;
+        }).when(antiAffinityGroupPolicyHelper).filter(any(), any());
         var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected2 = new HashSet<>();
         expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
@@ -675,6 +702,68 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
+    @Test
+    public void testFilterHasException() throws MetadataStoreException {
+        var filters = new ArrayList();
+        BrokerFilter filter = new BrokerFilter() {
+            @Override
+            public String name() {
+                return "Test-Filter";
+            }
+
+            @Override
+            public Map filter(Map brokers,
+                                                        ServiceUnitId serviceUnit,
+                                                        LoadManagerContext context) throws BrokerFilterException {
+                throw new BrokerFilterException("test");
+            }
+        };
+        filters.add(filter);
+        var counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, filters,
+                isolationPoliciesHelper, antiAffinityGroupPolicyHelper);
+
+        var ctx = setupContext();
+        ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
+    @Test
+    public void testIsLoadBalancerSheddingBundlesWithPoliciesEnabled() {
+        var counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, new ArrayList<>(),
+                isolationPoliciesHelper, antiAffinityGroupPolicyHelper);
+
+        var ctx = setupContext();
+
+        NamespaceBundle namespaceBundle = mock(NamespaceBundle.class);
+        doReturn("bundle").when(namespaceBundle).toString();
+
+        boolean[][] expects = {
+                {true, true, true, true},
+                {true, true, false, false},
+                {true, false, true, true},
+                {true, false, false, false},
+                {false, true, true, true},
+                {false, true, false, false},
+                {false, false, true, true},
+                {false, false, false, true}
+        };
+
+        for (boolean[] expect : expects) {
+            doReturn(expect[0]).when(isolationPoliciesHelper).hasIsolationPolicy(any());
+            doReturn(expect[1]).when(antiAffinityGroupPolicyHelper).hasAntiAffinityGroupPolicy(any());
+            ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(expect[2]);
+            assertEquals(transferShedder.isLoadBalancerSheddingBundlesWithPoliciesEnabled(ctx, namespaceBundle),
+                    expect[3]);
+        }
+    }
+
     @Test
     public void testTargetStd() {
         UnloadCounter counter = new UnloadCounter();

From f568c8f49828be42b8a7e81abea45a80cf4f93f4 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Wed, 5 Apr 2023 19:12:30 +0800
Subject: [PATCH 138/404] [improve][sql] Remove unnecessary future
 encapsulation (#19959)

Signed-off-by: tison 
Co-authored-by: tison 
---
 .../presto/PulsarSqlSchemaInfoProvider.java   | 33 ++++-----
 .../sql/presto/TestPulsarRecordCursor.java    | 67 ++++++++++---------
 2 files changed, 46 insertions(+), 54 deletions(-)

diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java
index d8f7db96b83d7..e2d030d2d7f1b 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java
@@ -27,8 +27,8 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
 import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.schema.SchemaInfoProvider;
 import org.apache.pulsar.common.naming.TopicName;
@@ -50,10 +50,13 @@ public class PulsarSqlSchemaInfoProvider implements SchemaInfoProvider {
 
     private final PulsarAdmin pulsarAdmin;
 
-    private final LoadingCache cache = CacheBuilder.newBuilder().maximumSize(100000)
-            .expireAfterAccess(30, TimeUnit.MINUTES).build(new CacheLoader() {
+    private final LoadingCache> cache = CacheBuilder.newBuilder()
+            .maximumSize(100000)
+            .expireAfterAccess(30, TimeUnit.MINUTES)
+            .build(new CacheLoader<>() {
+                @Nonnull
                 @Override
-                public SchemaInfo load(BytesSchemaVersion schemaVersion) throws Exception {
+                public CompletableFuture load(@Nonnull BytesSchemaVersion schemaVersion) {
                     return loadSchema(schemaVersion);
                 }
             });
@@ -69,7 +72,7 @@ public CompletableFuture getSchemaByVersion(byte[] schemaVersion) {
             if (null == schemaVersion) {
                 return completedFuture(null);
             }
-            return completedFuture(cache.get(BytesSchemaVersion.of(schemaVersion)));
+            return cache.get(BytesSchemaVersion.of(schemaVersion));
         } catch (ExecutionException e) {
             LOG.error("Can't get generic schema for topic {} schema version {}",
                     topicName.toString(), new String(schemaVersion, StandardCharsets.UTF_8), e);
@@ -79,14 +82,7 @@ public CompletableFuture getSchemaByVersion(byte[] schemaVersion) {
 
     @Override
     public CompletableFuture getLatestSchema() {
-        try {
-            return completedFuture(pulsarAdmin.schemas()
-                    .getSchemaInfo(topicName.toString()));
-        } catch (PulsarAdminException e) {
-            LOG.error("Can't get current schema for topic {}",
-                    topicName.toString(), e);
-            return FutureUtil.failedFuture(e.getCause());
-        }
+        return pulsarAdmin.schemas().getSchemaInfoAsync(topicName.toString());
     }
 
     @Override
@@ -94,24 +90,19 @@ public String getTopicName() {
         return topicName.getLocalName();
     }
 
-    private SchemaInfo loadSchema(BytesSchemaVersion bytesSchemaVersion) throws PulsarAdminException {
+    private CompletableFuture loadSchema(BytesSchemaVersion bytesSchemaVersion) {
         ClassLoader originalContextLoader = Thread.currentThread().getContextClassLoader();
         try {
             Thread.currentThread().setContextClassLoader(InjectionManagerFactory.class.getClassLoader());
             long version = ByteBuffer.wrap(bytesSchemaVersion.get()).getLong();
-            SchemaInfo schemaInfo = pulsarAdmin.schemas().getSchemaInfo(topicName.toString(), version);
-            if (schemaInfo == null) {
-                throw new RuntimeException(
-                        "The specific version (" + version + ") schema of the topic " + topicName + " is null");
-            }
-            return schemaInfo;
+            return pulsarAdmin.schemas().getSchemaInfoAsync(topicName.toString(), version);
         } finally {
             Thread.currentThread().setContextClassLoader(originalContextLoader);
         }
     }
 
 
-    public static SchemaInfo defaultSchema(){
+    public static SchemaInfo defaultSchema() {
         return Schema.BYTES.getSchemaInfo();
     }
 
diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java
index 7eaa2da498f45..40ced8e4f8e32 100644
--- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java
+++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java
@@ -18,6 +18,22 @@
  */
 package org.apache.pulsar.sql.presto;
 
+import static java.util.concurrent.CompletableFuture.completedFuture;
+import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.airlift.log.Logger;
 import io.netty.buffer.ByteBuf;
@@ -26,7 +42,18 @@
 import io.trino.spi.type.RowType;
 import io.trino.spi.type.Type;
 import io.trino.spi.type.VarcharType;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.math.BigDecimal;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import lombok.Data;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.Entry;
@@ -57,34 +84,6 @@
 import org.mockito.stubbing.Answer;
 import org.testng.annotations.Test;
 
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.concurrent.CompletableFuture.completedFuture;
-import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
 public class TestPulsarRecordCursor extends TestPulsarConnector {
 
     private static final Logger log = Logger.get(TestPulsarRecordCursor.class);
@@ -490,8 +489,8 @@ public void testGetSchemaInfo() throws Exception {
         // If the schemaVersion of the message is not null, try to get the schema.
         Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO);
         Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes());
-        Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L)))
-                .thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
+        Mockito.when(schemas.getSchemaInfoAsync(anyString(), eq(0L)))
+                .thenReturn(CompletableFuture.completedFuture(Schema.AVRO(Foo.class).getSchemaInfo()));
         schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
         assertEquals(SchemaType.AVRO, schemaInfo.getType());
 
@@ -516,19 +515,21 @@ public void testGetSchemaInfo() throws Exception {
 
         // If the specific version schema is null, throw runtime exception.
         Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes());
-        Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null);
+        Mockito.when(schemas.getSchemaInfoAsync(schemaTopic, 1))
+                .thenReturn(CompletableFuture.completedFuture(null));
         try {
             schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
             fail("The specific version " + 1 + " schema is null, should fail.");
         } catch (InvocationTargetException e) {
             String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString();
             assertTrue(e.getCause() instanceof RuntimeException);
-            assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null"));
+            assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null"));
         }
 
         // Get the specific version schema.
         Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes());
-        Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
+        Mockito.when(schemas.getSchemaInfoAsync(schemaTopic, 2))
+                .thenReturn(CompletableFuture.completedFuture(Schema.AVRO(Foo.class).getSchemaInfo()));
         schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
         assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
     }

From 836f03202dacd078c77dbe9dd2bcbf35f1d1a99e Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Wed, 5 Apr 2023 19:40:24 +0800
Subject: [PATCH 139/404] [fix][client] Fix DeadLetterProducer creation
 callback blocking client io thread. (#19930)

---
 .../java/org/apache/pulsar/client/impl/ConsumerImpl.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 1feef6ca0a642..74e6bf28baa1f 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -641,7 +641,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a
                 if (reconsumeTimes > this.deadLetterPolicy.getMaxRedeliverCount()
                         && StringUtils.isNotBlank(deadLetterPolicy.getDeadLetterTopic())) {
                     initDeadLetterProducerIfNeeded();
-                    deadLetterProducer.thenAccept(dlqProducer -> {
+                    deadLetterProducer.thenAcceptAsync(dlqProducer -> {
                         TypedMessageBuilder typedMessageBuilderNew =
                                 dlqProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(retryMessage.getReaderSchema().get()))
                                         .value(retryMessage.getData())
@@ -657,7 +657,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a
                             result.completeExceptionally(ex);
                             return null;
                         });
-                    }).exceptionally(ex -> {
+                    }, internalPinnedExecutor).exceptionally(ex -> {
                         result.completeExceptionally(ex);
                         deadLetterProducer = null;
                         return null;
@@ -2038,7 +2038,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId)
                                 return null;
                     });
                 }
-            }).exceptionally(ex -> {
+            }, internalPinnedExecutor).exceptionally(ex -> {
                 log.error("Dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), ex);
                 deadLetterProducer = null;
                 result.complete(false);

From 32e677d5264d9145a5765735ae1e6ba0e88a5ef9 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Wed, 5 Apr 2023 05:00:32 -0700
Subject: [PATCH 140/404] [improve][broker] PIP-192 updated metrics and cleanup
 broker selector (#19945)

---
 .../pulsar/PulsarClusterMetadataSetup.java    |   4 +-
 .../extensions/ExtensibleLoadManagerImpl.java |  27 +--
 .../channel/ServiceUnitStateChannelImpl.java  | 197 +++++++++++++-----
 .../extensions/models/AssignCounter.java      |  10 +-
 ...faultNamespaceBundleSplitStrategyImpl.java |   3 +-
 .../ExtensibleLoadManagerImplTest.java        |  93 +++------
 .../channel/ServiceUnitStateChannelTest.java  |  35 ++--
 ...faultNamespaceBundleSplitStrategyTest.java |   2 +-
 8 files changed, 212 insertions(+), 159 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
index 1a2ca1ec4fb53..0badbda1afdfd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
@@ -392,8 +392,8 @@ static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName nam
         }
     }
 
-    static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName,
-            String cluster) throws IOException {
+    public static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName,
+                                               String cluster) throws IOException {
         createNamespaceIfAbsent(resources, namespaceName, cluster, DEFAULT_BUNDLE_NUMBER);
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 7e84fa5969a6d..c1234b6dab21e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -164,8 +164,6 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
     private final UnloadCounter unloadCounter = new UnloadCounter();
     private final SplitCounter splitCounter = new SplitCounter();
 
-    // record load metrics
-    private final AtomicReference> brokerLoadMetrics = new AtomicReference<>();
     // record unload metrics
     private final AtomicReference> unloadMetrics = new AtomicReference();
     // record split metrics
@@ -338,7 +336,6 @@ public CompletableFuture> assign(Optional> assign(Optional lookupRequests.remove(bundle));
+        future.whenComplete((r, t) -> {
+                    if (t != null) {
+                        assignCounter.incrementFailure();
+                    }
+                    lookupRequests.remove(bundle);
+                }
+        );
         return future;
     }
 
-    private CompletableFuture> selectAsync(ServiceUnitId bundle) {
+    public CompletableFuture> selectAsync(ServiceUnitId bundle) {
         BrokerRegistry brokerRegistry = getBrokerRegistry();
         return brokerRegistry.getAvailableBrokerLookupDataAsync()
                 .thenCompose(availableBrokers -> {
@@ -633,20 +636,12 @@ void playFollower() {
         }
     }
 
-    void updateBrokerLoadMetrics(BrokerLoadData loadData) {
-        this.brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress()));
-    }
-
-    private void updateUnloadMetrics(UnloadDecision decision) {
-        unloadCounter.update(decision);
-        this.unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress()));
-    }
-
     public List getMetrics() {
         List metricsCollection = new ArrayList<>();
 
-        if (this.brokerLoadMetrics.get() != null) {
-            metricsCollection.addAll(this.brokerLoadMetrics.get());
+        if (this.brokerLoadDataReporter != null) {
+            metricsCollection.addAll(brokerLoadDataReporter.generateLoadData()
+                    .toMetrics(pulsar.getAdvertisedAddress()));
         }
         if (this.unloadMetrics.get() != null) {
             metricsCollection.addAll(this.unloadMetrics.get());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index e7fb59450b25c..ec79698db1f05 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -41,6 +41,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Stable;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state;
+import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE;
 import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost;
 import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished;
 import com.google.common.annotations.VisibleForTesting;
@@ -65,6 +66,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.pulsar.PulsarClusterMetadataSetup;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -76,8 +78,6 @@
 import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
-import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
-import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
@@ -104,7 +104,7 @@
 public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     public static final String TOPIC = TopicName.get(
             TopicDomain.persistent.value(),
-            NamespaceName.SYSTEM_NAMESPACE,
+            SYSTEM_NAMESPACE,
             "loadbalancer-service-unit-state").toString();
     private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec
     public static final long VERSION_ID_INIT = 1; // initial versionId
@@ -113,6 +113,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately
     private static final long MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS = 10;
     private static final int MAX_OUTSTANDING_PUB_MESSAGES = 500;
+    private static final long MAX_OWNED_BUNDLE_COUNT_DELAY_TIME_IN_MILLIS = 10 * 60 * 1000;
     private final PulsarService pulsar;
     private final ServiceConfiguration config;
     private final Schema schema;
@@ -120,7 +121,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     private final String lookupServiceAddress;
     private final ConcurrentOpenHashMap> cleanupJobs;
     private final StateChangeListeners stateChangeListeners;
-    private BrokerSelectionStrategy brokerSelector;
+    private ExtensibleLoadManagerImpl loadManager;
     private BrokerRegistry brokerRegistry;
     private LeaderElectionService leaderElectionService;
     private TableView tableview;
@@ -144,11 +145,15 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     private long totalInactiveBrokerCleanupIgnoredCnt = 0;
     private long totalInactiveBrokerCleanupCancelledCnt = 0;
     private volatile ChannelState channelState;
+    private volatile long lastOwnEventHandledAt = 0;
+    private long lastOwnedServiceUnitCountAt = 0;
+    private int totalOwnedServiceUnitCnt = 0;
 
     public enum EventType {
         Assign,
         Split,
-        Unload
+        Unload,
+        Override
 
     }
 
@@ -164,7 +169,7 @@ public Counters(){
     }
 
     // operation metrics
-    final Map ownerLookUpCounters;
+    final Map ownerLookUpCounters;
     final Map eventCounters;
     final Map handlerCounters;
 
@@ -207,11 +212,11 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) {
         this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS;
         this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS;
 
-        Map tmpOwnerLookUpCounters = new HashMap<>();
+        Map tmpOwnerLookUpCounters = new HashMap<>();
         Map tmpHandlerCounters = new HashMap<>();
         Map tmpEventCounters = new HashMap<>();
         for (var state : ServiceUnitState.values()) {
-            tmpOwnerLookUpCounters.put(state, new AtomicLong());
+            tmpOwnerLookUpCounters.put(state, new Counters());
             tmpHandlerCounters.put(state, new Counters());
         }
         for (var event : EventType.values()) {
@@ -267,7 +272,7 @@ public synchronized void start() throws PulsarServerException {
                 log.warn("Failed to find the channel leader.");
             }
             this.channelState = LeaderElectionServiceStarted;
-            brokerSelector = getBrokerSelector();
+            loadManager = getLoadManager();
 
             if (producer != null) {
                 producer.close();
@@ -275,6 +280,9 @@ public synchronized void start() throws PulsarServerException {
                     log.info("Closed the channel producer.");
                 }
             }
+            PulsarClusterMetadataSetup.createNamespaceIfAbsent
+                    (pulsar.getPulsarResources(), NamespaceName.SYSTEM_NAMESPACE, config.getClusterName());
+
             producer = pulsar.getClient().newProducer(schema)
                     .enableBatching(true)
                     .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES)
@@ -329,9 +337,8 @@ protected LoadManagerContext getContext() {
     }
 
     @VisibleForTesting
-    protected BrokerSelectionStrategy getBrokerSelector() {
-        // TODO: make this selector configurable.
-        return new LeastResourceUsageWithWeight();
+    protected ExtensibleLoadManagerImpl getLoadManager() {
+        return ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get());
     }
 
     @VisibleForTesting
@@ -465,7 +472,7 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) {
 
         ServiceUnitStateData data = tableview.get(serviceUnit);
         ServiceUnitState state = state(data);
-        ownerLookUpCounters.get(state).incrementAndGet();
+        ownerLookUpCounters.get(state).getTotal().incrementAndGet();
         switch (state) {
             case Owned -> {
                 return CompletableFuture.completedFuture(Optional.of(data.dstBroker()));
@@ -474,16 +481,22 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) {
                 return CompletableFuture.completedFuture(Optional.of(data.sourceBroker()));
             }
             case Assigning, Releasing -> {
-                return deferGetOwnerRequest(serviceUnit).thenApply(
+                return deferGetOwnerRequest(serviceUnit).whenComplete((__, e) -> {
+                    if (e != null) {
+                        ownerLookUpCounters.get(state).getFailure().incrementAndGet();
+                    }
+                }).thenApply(
                         broker -> broker == null ? Optional.empty() : Optional.of(broker));
             }
             case Init, Free -> {
                 return CompletableFuture.completedFuture(Optional.empty());
             }
             case Deleted -> {
+                ownerLookUpCounters.get(state).getFailure().incrementAndGet();
                 return CompletableFuture.failedFuture(new IllegalArgumentException(serviceUnit + " is deleted."));
             }
             default -> {
+                ownerLookUpCounters.get(state).getFailure().incrementAndGet();
                 String errorMsg = String.format("Failed to process service unit state data: %s when get owner.", data);
                 log.error(errorMsg);
                 return CompletableFuture.failedFuture(new IllegalStateException(errorMsg));
@@ -522,6 +535,23 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str
         return getOwnerRequest;
     }
 
+    private CompletableFuture publishOverrideEventAsync(String serviceUnit,
+                                           ServiceUnitStateData orphanData,
+                                           ServiceUnitStateData override) {
+        if (!validateChannelState(Started, true)) {
+            throw new IllegalStateException("Invalid channel state:" + channelState.name());
+        }
+        EventType eventType = EventType.Override;
+        eventCounters.get(eventType).getTotal().incrementAndGet();
+        return pubAsync(serviceUnit, override).whenComplete((__, e) -> {
+            if (e != null) {
+                eventCounters.get(eventType).getFailure().incrementAndGet();
+                log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}",
+                        serviceUnit, orphanData, override, e);
+            }
+        }).thenApply(__ -> null);
+    }
+
     public CompletableFuture publishUnloadEventAsync(Unload unload) {
         if (!validateChannelState(Started, true)) {
             return CompletableFuture.failedFuture(
@@ -658,6 +688,7 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) {
         stateChangeListeners.notify(serviceUnit, data, null);
         if (isTargetBroker(data.dstBroker())) {
             log(null, serviceUnit, data, null);
+            lastOwnEventHandledAt = System.currentTimeMillis();
         }
     }
 
@@ -1022,33 +1053,34 @@ private void scheduleCleanup(String broker, long delayInSecs) {
                 broker, delayInSecs, cleanupJobs.size());
     }
 
-    private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, Set availableBrokers) {
-
-        Optional selectedBroker = brokerSelector.select(availableBrokers, null, getContext());
+    private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) {
+        Optional selectedBroker = selectBroker(serviceUnit);
         if (selectedBroker.isPresent()) {
-            var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData));
+            var override =
+                    new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData));
             log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}",
                     serviceUnit, orphanData, override);
-            pubAsync(serviceUnit, override).whenComplete((__, e) -> {
-                if (e != null) {
-                    log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}",
-                            serviceUnit, orphanData, override, e);
-                }
-            });
+            publishOverrideEventAsync(serviceUnit, orphanData, override)
+                    .exceptionally(e -> {
+                        log.error(
+                                "Failed to override the ownership serviceUnit:{} orphanData:{}. "
+                                        + "Failed to publish override event. totalCleanupErrorCnt:{}",
+                                serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet());
+                        return null;
+                    });
         } else {
-            log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker.",
-                    serviceUnit, orphanData);
+            log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker. "
+                            + "totalCleanupErrorCnt:{}",
+                    serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet());
         }
     }
 
 
-    private void doCleanup(String broker) throws ExecutionException, InterruptedException, TimeoutException {
+    private void doCleanup(String broker)  {
         long startTime = System.nanoTime();
         log.info("Started ownership cleanup for the inactive broker:{}", broker);
         int orphanServiceUnitCleanupCnt = 0;
         long totalCleanupErrorCntStart = totalCleanupErrorCnt.get();
-        var availableBrokers = new HashSet<>(brokerRegistry.getAvailableBrokersAsync()
-                .get(inFlightStateWaitingTimeInMillis, MILLISECONDS));
 
         for (var etr : tableview.entrySet()) {
             var stateData = etr.getValue();
@@ -1056,13 +1088,13 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce
             var state = state(stateData);
             if (StringUtils.equals(broker, stateData.dstBroker())) {
                 if (isActiveState(state)) {
-                    overrideOwnership(serviceUnit, stateData, availableBrokers);
+                    overrideOwnership(serviceUnit, stateData);
                     orphanServiceUnitCleanupCnt++;
                 }
 
             } else if (StringUtils.equals(broker, stateData.sourceBroker())) {
                 if (isInFlightState(state)) {
-                    overrideOwnership(serviceUnit, stateData, availableBrokers);
+                    overrideOwnership(serviceUnit, stateData);
                     orphanServiceUnitCleanupCnt++;
                 }
             }
@@ -1097,31 +1129,40 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce
 
     }
 
-    private Optional getRollForwardStateData(
-            Set availableBrokers, LoadManagerContext context, long nextVersionId) {
-        Optional selectedBroker = brokerSelector.select(availableBrokers, null, context);
+    private Optional selectBroker(String serviceUnit) {
+        try {
+            return loadManager.selectAsync(getNamespaceBundle(serviceUnit))
+                    .get(inFlightStateWaitingTimeInMillis, MILLISECONDS);
+        } catch (Throwable e) {
+            log.error("Failed to select a broker for serviceUnit:{}", serviceUnit);
+        }
+        return Optional.empty();
+    }
+
+    private Optional getRollForwardStateData(String serviceUnit, long nextVersionId) {
+        Optional selectedBroker = selectBroker(serviceUnit);
         if (selectedBroker.isEmpty()) {
             return Optional.empty();
         }
         return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId));
     }
 
+
     private Optional getOverrideInFlightStateData(
             String serviceUnit, ServiceUnitStateData orphanData,
-            Set availableBrokers,
-            LoadManagerContext context) {
+            Set availableBrokers) {
         long nextVersionId = getNextVersionId(orphanData);
         var state = orphanData.state();
         switch (state) {
             case Assigning: {
-                return getRollForwardStateData(availableBrokers, context, nextVersionId);
+                return getRollForwardStateData(serviceUnit, nextVersionId);
             }
             case Splitting, Releasing: {
                 if (availableBrokers.contains(orphanData.sourceBroker())) {
                     // rollback to the src
                     return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId));
                 } else {
-                    return getRollForwardStateData(availableBrokers, context, nextVersionId);
+                    return getRollForwardStateData(serviceUnit, nextVersionId);
                 }
             }
             default: {
@@ -1202,25 +1243,27 @@ protected void monitorOwnerships(List brokers) {
                 handleBrokerDeletionEvent(inactiveBroker);
             }
         } else if (!orphanServiceUnits.isEmpty()) {
-            var context = getContext();
             for (var etr : orphanServiceUnits.entrySet()) {
                 var orphanServiceUnit = etr.getKey();
                 var orphanData = etr.getValue();
                 var overrideData = getOverrideInFlightStateData(
-                        orphanServiceUnit, orphanData, activeBrokers, context);
+                        orphanServiceUnit, orphanData, activeBrokers);
                 if (overrideData.isPresent()) {
-                    pubAsync(orphanServiceUnit, overrideData.get()).whenComplete((__, e) -> {
-                        if (e != null) {
-                            log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, "
-                                            + "cleanupErrorCnt:{}.",
-                                    orphanServiceUnit, orphanData,
-                                    totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e);
-                        }
-                    });
+                    publishOverrideEventAsync(orphanServiceUnit, orphanData, overrideData.get())
+                            .whenComplete((__, e) -> {
+                                if (e != null) {
+                                    log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, "
+                                                    + "cleanupErrorCnt:{}.",
+                                            orphanServiceUnit, orphanData,
+                                            totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e);
+                                }
+                            });
                     orphanServiceUnitCleanupCnt++;
                 } else {
-                    log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{}. will retry..",
-                            orphanServiceUnit, orphanData);
+                    log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{},"
+                                    + " cleanupErrorCnt:{}. will retry..",
+                            orphanServiceUnit, orphanData,
+                            totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart);
                 }
             }
         }
@@ -1279,6 +1322,25 @@ private String printCleanupMetrics() {
         );
     }
 
+    private int getTotalOwnedServiceUnitCnt() {
+        if (tableview == null) {
+            return 0;
+        }
+        long now = System.currentTimeMillis();
+        if (lastOwnEventHandledAt > lastOwnedServiceUnitCountAt
+                || now - lastOwnedServiceUnitCountAt > MAX_OWNED_BUNDLE_COUNT_DELAY_TIME_IN_MILLIS) {
+            int cnt = 0;
+            for (var data : tableview.values()) {
+                if (data.state() == Owned && isTargetBroker(data.dstBroker())) {
+                    cnt++;
+                }
+            }
+            lastOwnedServiceUnitCountAt = now;
+            totalOwnedServiceUnitCnt = cnt;
+        }
+        return totalOwnedServiceUnitCnt;
+    }
+
 
     @Override
     public List getMetrics() {
@@ -1288,11 +1350,25 @@ public List getMetrics() {
         dimensions.put("broker", pulsar.getAdvertisedAddress());
 
         for (var etr : ownerLookUpCounters.entrySet()) {
-            var dim = new HashMap<>(dimensions);
-            dim.put("state", etr.getKey().toString());
-            var metric = Metrics.create(dim);
-            metric.put("brk_sunit_state_chn_owner_lookup_total", etr.getValue());
-            metrics.add(metric);
+            {
+                var dim = new HashMap<>(dimensions);
+                dim.put("state", etr.getKey().toString());
+                dim.put("result", "Total");
+                var metric = Metrics.create(dim);
+                metric.put("brk_sunit_state_chn_owner_lookup_total",
+                        etr.getValue().getTotal().get());
+                metrics.add(metric);
+            }
+
+            {
+                var dim = new HashMap<>(dimensions);
+                dim.put("state", etr.getKey().toString());
+                dim.put("result", "Failure");
+                var metric = Metrics.create(dim);
+                metric.put("brk_sunit_state_chn_owner_lookup_total",
+                        etr.getValue().getFailure().get());
+                metrics.add(metric);
+            }
         }
 
         for (var etr : eventCounters.entrySet()) {
@@ -1371,10 +1447,19 @@ public List getMetrics() {
             metrics.add(metric);
         }
 
+        {
+            var dim = new HashMap<>(dimensions);
+            dim.put("result", "Success");
+            var metric = Metrics.create(dim);
+            metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCnt);
+            metrics.add(metric);
+        }
+
         var metric = Metrics.create(dimensions);
-        metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCnt);
+
         metric.put("brk_sunit_state_chn_orphan_su_cleanup_ops_total", totalOrphanServiceUnitCleanupCnt);
         metric.put("brk_sunit_state_chn_su_tombstone_cleanup_ops_total", totalServiceUnitTombstoneCleanupCnt);
+        metric.put("brk_sunit_state_chn_owned_su_total", getTotalOwnedServiceUnitCnt());
         metrics.add(metric);
 
         return metrics;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java
index 26ff1f5f401d1..8e19b7b45e2d4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java
@@ -18,7 +18,7 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
-import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Empty;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Success;
 import java.util.ArrayList;
@@ -35,7 +35,7 @@ public class AssignCounter {
 
     enum Label {
         Success,
-        Empty,
+        Failure,
         Skip,
     }
 
@@ -44,7 +44,7 @@ enum Label {
     public AssignCounter() {
         breakdownCounters = Map.of(
                 Success, new AtomicLong(),
-                Empty, new AtomicLong(),
+                Failure, new AtomicLong(),
                 Skip, new AtomicLong()
         );
     }
@@ -54,8 +54,8 @@ public void incrementSuccess() {
         breakdownCounters.get(Success).incrementAndGet();
     }
 
-    public void incrementEmpty() {
-        breakdownCounters.get(Empty).incrementAndGet();
+    public void incrementFailure() {
+        breakdownCounters.get(Failure).incrementAndGet();
     }
 
     public void incrementSkip() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
index 15bfdc747f1fc..bd8b3aa66542a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -98,9 +98,8 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
 
             if (!channel.isOwner(bundle)) {
                 if (debug) {
-                    log.error(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                    log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG
                             + " This broker is not the owner.", bundle));
-                    counter.update(Failure, Unknown);
                 }
                 continue;
             }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 502e02e465e64..a9b145e30293c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -37,6 +37,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
@@ -59,7 +60,6 @@
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import java.net.URL;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -81,13 +81,11 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.namespace.LookupOptions;
-import org.apache.pulsar.broker.resources.NamespaceResources;
-import org.apache.pulsar.broker.resources.PulsarResources;
-import org.apache.pulsar.broker.resources.TenantResources;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.impl.TableViewImpl;
@@ -97,11 +95,8 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.BundlesData;
 import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.Policies;
-import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.apache.pulsar.common.stats.Metrics;
-import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
@@ -122,8 +117,6 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest {
 
     private PulsarTestContext additionalPulsarTestContext;
 
-    private PulsarResources resources;
-
     private ExtensibleLoadManagerImpl primaryLoadManager;
 
     private ExtensibleLoadManagerImpl secondaryLoadManager;
@@ -164,37 +157,6 @@ public void setup() throws Exception {
                 Sets.newHashSet(this.conf.getClusterName()));
     }
 
-    protected void beforePulsarStart(PulsarService pulsar) throws Exception {
-        if (resources == null) {
-            MetadataStoreExtended localStore = pulsar.createLocalMetadataStore(null);
-            MetadataStoreExtended configStore = (MetadataStoreExtended) pulsar.createConfigurationMetadataStore(null);
-            resources = new PulsarResources(localStore, configStore);
-        }
-        this.createNamespaceIfNotExists(resources, NamespaceName.SYSTEM_NAMESPACE.getTenant(),
-                NamespaceName.SYSTEM_NAMESPACE);
-    }
-
-    protected void createNamespaceIfNotExists(PulsarResources resources,
-                                              String publicTenant,
-                                              NamespaceName ns) throws Exception {
-        TenantResources tr = resources.getTenantResources();
-        NamespaceResources nsr = resources.getNamespaceResources();
-
-        if (!tr.tenantExists(publicTenant)) {
-            tr.createTenant(publicTenant,
-                    TenantInfo.builder()
-                            .adminRoles(Sets.newHashSet(conf.getSuperUserRoles()))
-                            .allowedClusters(Sets.newHashSet(conf.getClusterName()))
-                            .build());
-        }
-
-        if (!nsr.namespaceExists(ns)) {
-            Policies nsp = new Policies();
-            nsp.replication_clusters = Collections.singleton(conf.getClusterName());
-            nsr.createPolicies(ns, nsp);
-        }
-    }
-
     @Override
     @AfterClass
     protected void cleanup() throws Exception {
@@ -635,8 +597,8 @@ public void testRoleChange()
     @Test
     public void testGetMetrics() throws Exception {
         {
-            var brokerLoadMetrics = (AtomicReference>)
-                    FieldUtils.readDeclaredField(primaryLoadManager, "brokerLoadMetrics", true);
+            var brokerLoadDataReporter = mock(BrokerLoadDataReporter.class);
+            FieldUtils.writeDeclaredField(primaryLoadManager, "brokerLoadDataReporter", brokerLoadDataReporter, true);
             BrokerLoadData loadData = new BrokerLoadData();
             SystemResourceUsage usage = new SystemResourceUsage();
             var cpu = new ResourceUsage(1.0, 100.0);
@@ -650,7 +612,7 @@ public void testGetMetrics() throws Exception {
             usage.setBandwidthIn(bandwidthIn);
             usage.setBandwidthOut(bandwidthOut);
             loadData.update(usage, 1, 2, 3, 4, 5, 6, conf);
-            brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress()));
+            doReturn(loadData).when(brokerLoadDataReporter).generateLoadData();
         }
         {
             var unloadMetrics = (AtomicReference>)
@@ -700,8 +662,8 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
         {
             AssignCounter assignCounter = new AssignCounter();
             assignCounter.incrementSuccess();
-            assignCounter.incrementEmpty();
-            assignCounter.incrementEmpty();
+            assignCounter.incrementFailure();
+            assignCounter.incrementFailure();
             assignCounter.incrementSkip();
             assignCounter.incrementSkip();
             assignCounter.incrementSkip();
@@ -709,7 +671,8 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
         }
 
         {
-
+            FieldUtils.writeDeclaredField(channel1, "lastOwnedServiceUnitCountAt", System.currentTimeMillis(), true);
+            FieldUtils.writeDeclaredField(channel1, "totalOwnedServiceUnitCnt", 10, true);
             FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCnt", 1, true);
             FieldUtils.writeDeclaredField(channel1, "totalServiceUnitTombstoneCleanupCnt", 2, true);
             FieldUtils.writeDeclaredField(channel1, "totalOrphanServiceUnitCleanupCnt", 3, true);
@@ -718,21 +681,21 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
             FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupIgnoredCnt", 6, true);
             FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCancelledCnt", 7, true);
 
-            Map ownerLookUpCounters = new LinkedHashMap<>();
+            Map ownerLookUpCounters = new LinkedHashMap<>();
             Map handlerCounters = new LinkedHashMap<>();
             Map eventCounters =
                     new LinkedHashMap<>();
-            int i = 1;
             int j = 0;
             for (var state : ServiceUnitState.values()) {
-                ownerLookUpCounters.put(state, new AtomicLong(i));
+                ownerLookUpCounters.put(state,
+                        new ServiceUnitStateChannelImpl.Counters(
+                                new AtomicLong(j + 1), new AtomicLong(j + 2)));
                 handlerCounters.put(state,
                         new ServiceUnitStateChannelImpl.Counters(
                                 new AtomicLong(j + 1), new AtomicLong(j + 2)));
-                i++;
                 j += 2;
             }
-            i = 0;
+            int i = 0;
             for (var type : ServiceUnitStateChannelImpl.EventType.values()) {
                 eventCounters.put(type,
                         new ServiceUnitStateChannelImpl.Counters(
@@ -769,22 +732,31 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}]
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}]
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=6}]
-                        dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}]
+                        dimensions=[{broker=localhost, metric=assign, result=Failure}], metrics=[{brk_lb_assign_broker_breakdown_total=2}]
                         dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=6}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=7}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=6}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=7}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=8}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=9}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=10}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=11}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=12}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Total, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=13}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=14}]
                         dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=1}]
                         dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=2}]
                         dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}]
                         dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}]
                         dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}]
                         dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}]
+                        dimensions=[{broker=localhost, event=Override, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=7}]
+                        dimensions=[{broker=localhost, event=Override, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=8}]
                         dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}]
                         dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}]
                         dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}]
@@ -803,7 +775,8 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
                         dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=6}]
                         dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=7}]
                         dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=5}]
-                        dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1, brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn, result=Success}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1}]
+                        dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_owned_su_total=10, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}]
                         """.split("\n"));
         var actual = primaryLoadManager.getMetrics().stream().map(m -> m.toString()).collect(Collectors.toSet());
         assertEquals(actual, expected);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 9eda98e5d842d..2d98bc5cae6e5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -75,11 +75,11 @@
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistryImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
-import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.api.Producer;
@@ -116,7 +116,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
 
     private BrokerRegistryImpl registry;
 
-    private BrokerSelectionStrategy brokerSelector;
+    private ExtensibleLoadManagerImpl loadManager;
 
     @BeforeClass
     @Override
@@ -135,7 +135,7 @@ protected void setup() throws Exception {
         loadManagerContext = mock(LoadManagerContext.class);
         doReturn(mock(LoadDataStore.class)).when(loadManagerContext).brokerLoadDataStore();
         doReturn(mock(LoadDataStore.class)).when(loadManagerContext).topBundleLoadDataStore();
-        brokerSelector = mock(BrokerSelectionStrategy.class);
+        loadManager = mock(ExtensibleLoadManagerImpl.class);
         additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf());
         pulsar2 = additionalPulsarTestContext.getPulsarService();
 
@@ -496,7 +496,7 @@ public void transferTestWhenDestBrokerFails()
         assertEquals(0, getOwnerRequests2.size());
 
         // recovered, check the monitor update state : Assigned -> Owned
-        doReturn(Optional.of(lookupServiceAddress1)).when(brokerSelector).select(any(), any(), any());
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))).when(loadManager).selectAsync(any());
         FieldUtils.writeDeclaredField(channel2, "producer", producer, true);
         FieldUtils.writeDeclaredField(channel1,
                 "inFlightStateWaitingTimeInMillis", 1 , true);
@@ -714,7 +714,7 @@ public void handleBrokerDeletionEventTest()
 
         var owner1 = channel1.getOwnerAsync(bundle1);
         var owner2 = channel2.getOwnerAsync(bundle2);
-        doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any());
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any());
 
         assertTrue(owner1.get().isEmpty());
         assertTrue(owner2.get().isEmpty());
@@ -1076,7 +1076,7 @@ public void assignTestWhenDestBrokerFails()
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
         FieldUtils.writeDeclaredField(channel2,
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
-        doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any());
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any());
         channel1.publishAssignEventAsync(bundle, lookupServiceAddress2);
         // channel1 is broken. the assign won't be complete.
         waitUntilState(channel1, bundle);
@@ -1418,11 +1418,12 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel)
         }
 
         var ownerLookUpCounters =
-                (Map)
+                (Map)
                         FieldUtils.readDeclaredField(channel, "ownerLookUpCounters", true);
 
         for(var val : ownerLookUpCounters.values()){
-            val.set(0);
+            val.getFailure().set(0);
+            val.getTotal().set(0);
         }
     }
 
@@ -1518,20 +1519,20 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel,
                                                     )
             throws IllegalAccessException {
         var ownerLookUpCounters =
-                (Map)
+                (Map)
                         FieldUtils.readDeclaredField(channel, "ownerLookUpCounters", true);
 
         Awaitility.await()
                 .pollInterval(200, TimeUnit.MILLISECONDS)
                 .atMost(10, TimeUnit.SECONDS)
                 .untilAsserted(() -> { // wait until true
-                    assertEquals(assigned, ownerLookUpCounters.get(Assigning).get());
-                    assertEquals(owned, ownerLookUpCounters.get(Owned).get());
-                    assertEquals(released, ownerLookUpCounters.get(Releasing).get());
-                    assertEquals(splitting, ownerLookUpCounters.get(Splitting).get());
-                    assertEquals(free, ownerLookUpCounters.get(Free).get());
-                    assertEquals(deleted, ownerLookUpCounters.get(Deleted).get());
-                    assertEquals(init, ownerLookUpCounters.get(Init).get());
+                    assertEquals(assigned, ownerLookUpCounters.get(Assigning).getTotal().get());
+                    assertEquals(owned, ownerLookUpCounters.get(Owned).getTotal().get());
+                    assertEquals(released, ownerLookUpCounters.get(Releasing).getTotal().get());
+                    assertEquals(splitting, ownerLookUpCounters.get(Splitting).getTotal().get());
+                    assertEquals(free, ownerLookUpCounters.get(Free).getTotal().get());
+                    assertEquals(deleted, ownerLookUpCounters.get(Deleted).getTotal().get());
+                    assertEquals(init, ownerLookUpCounters.get(Init).getTotal().get());
                 });
     }
 
@@ -1565,7 +1566,7 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar)
 
         doReturn(loadManagerContext).when(channel).getContext();
         doReturn(registry).when(channel).getBrokerRegistry();
-        doReturn(brokerSelector).when(channel).getBrokerSelector();
+        doReturn(loadManager).when(channel).getLoadManager();
 
 
         var leaderElectionService = new LeaderElectionService(
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
index 0aa055b58acd3..8c765e7a3df64 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -169,7 +169,7 @@ public void testNoBundleOwner() {
         var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
         var expected = Set.of();
         assertEquals(actual, expected);
-        verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+        verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
     }
 
     public void testError() throws Exception {

From 0c9a866f948ed6636050fac110563ad4e64bb3b1 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 5 Apr 2023 16:21:33 +0300
Subject: [PATCH 141/404] [improve][proxy] Implement graceful shutdown for
 Pulsar Proxy (#20011)

---
 .../pulsar/proxy/server/ProxyService.java     | 76 +++++++++++++++----
 1 file changed, 63 insertions(+), 13 deletions(-)

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
index dfe888d06510e..4cca24f5f4892 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.pulsar.proxy.server;
 
 import static java.util.Objects.requireNonNull;
@@ -188,7 +189,7 @@ public ProxyService(ProxyConfiguration proxyConfig,
 
         statsExecutor = Executors
                 .newSingleThreadScheduledExecutor(new DefaultThreadFactory("proxy-stats-executor"));
-        statsExecutor.schedule(()->{
+        statsExecutor.schedule(() -> {
             this.clientCnxs.forEach(cnx -> {
                 if (cnx.getDirectProxyHandler() != null
                         && cnx.getDirectProxyHandler().getInboundChannelRequestsRate() != null) {
@@ -223,7 +224,7 @@ public void start() throws Exception {
             pulsarResources = new PulsarResources(localMetadataStore, configMetadataStore);
             discoveryProvider = new BrokerDiscoveryProvider(this.proxyConfig, pulsarResources);
             authorizationService = new AuthorizationService(PulsarConfigurationLoader.convertFrom(proxyConfig),
-                                                            pulsarResources);
+                    pulsarResources);
         }
 
         ServerBootstrap bootstrap = new ServerBootstrap();
@@ -265,7 +266,7 @@ public void start() throws Exception {
         }
 
         final String hostname =
-            ServiceConfigurationUtils.getDefaultOrConfiguredAddress(proxyConfig.getAdvertisedAddress());
+                ServiceConfigurationUtils.getDefaultOrConfiguredAddress(proxyConfig.getAdvertisedAddress());
 
         if (proxyConfig.getServicePort().isPresent()) {
             this.serviceUrl = String.format("pulsar://%s:%d/", hostname, getListenPort().get());
@@ -352,18 +353,38 @@ public BrokerDiscoveryProvider getDiscoveryProvider() {
     }
 
     public void close() throws IOException {
-        dnsAddressResolverGroup.close();
+        if (listenChannel != null) {
+            try {
+                listenChannel.close().sync();
+            } catch (InterruptedException e) {
+                LOG.info("Shutdown of listenChannel interrupted");
+                Thread.currentThread().interrupt();
+            }
+        }
 
-        if (discoveryProvider != null) {
-            discoveryProvider.close();
+        if (listenChannelTls != null) {
+            try {
+                listenChannelTls.close().sync();
+            } catch (InterruptedException e) {
+                LOG.info("Shutdown of listenChannelTls interrupted");
+                Thread.currentThread().interrupt();
+            }
         }
 
-        if (listenChannel != null) {
-            listenChannel.close();
+        // Don't accept any new connections
+        try {
+            acceptorGroup.shutdownGracefully().sync();
+        } catch (InterruptedException e) {
+            LOG.info("Shutdown of acceptorGroup interrupted");
+            Thread.currentThread().interrupt();
         }
 
-        if (listenChannelTls != null) {
-            listenChannelTls.close();
+        closeAllConnections();
+
+        dnsAddressResolverGroup.close();
+
+        if (discoveryProvider != null) {
+            discoveryProvider.close();
         }
 
         if (statsExecutor != null) {
@@ -391,10 +412,39 @@ public void close() throws IOException {
                 throw new IOException(e);
             }
         }
-        acceptorGroup.shutdownGracefully();
-        workerGroup.shutdownGracefully();
+        try {
+            workerGroup.shutdownGracefully().sync();
+        } catch (InterruptedException e) {
+            LOG.info("Shutdown of workerGroup interrupted");
+            Thread.currentThread().interrupt();
+        }
         for (EventLoopGroup group : extensionsWorkerGroups) {
-            group.shutdownGracefully();
+            try {
+                group.shutdownGracefully().sync();
+            } catch (InterruptedException e) {
+                LOG.info("Shutdown of {} interrupted", group);
+                Thread.currentThread().interrupt();
+            }
+        }
+        LOG.info("ProxyService closed.");
+    }
+
+    private void closeAllConnections() {
+        try {
+            workerGroup.submit(() -> {
+                // Close all the connections
+                if (!clientCnxs.isEmpty()) {
+                    LOG.info("Closing {} proxy connections, including connections to brokers", clientCnxs.size());
+                    for (ProxyConnection clientCnx : clientCnxs) {
+                        clientCnx.ctx().close();
+                    }
+                } else {
+                    LOG.info("No proxy connections to close");
+                }
+            }).sync();
+        } catch (InterruptedException e) {
+            LOG.info("Closing of connections interrupted");
+            Thread.currentThread().interrupt();
         }
     }
 

From be7e890d91cbb28014280b088de35b3a3bf513b8 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Wed, 5 Apr 2023 22:32:48 +0800
Subject: [PATCH 142/404] [improve][broker] PIP-192: Update the lookup data
 path to support deploy and rollback (#19999)

Co-authored-by: Jiwe Guo 
---
 .../extensions/BrokerRegistryImpl.java        |  2 +-
 .../ExtensibleLoadManagerImplTest.java        | 30 ++++++++++++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java
index de0d361316d8d..4c2beba5d30b3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java
@@ -51,7 +51,7 @@
 @Slf4j
 public class BrokerRegistryImpl implements BrokerRegistry {
 
-    protected static final String LOOKUP_DATA_PATH = "/loadbalance/brokers";
+    protected static final String LOOKUP_DATA_PATH = "/loadbalance/extension/brokers";
 
     private final PulsarService pulsar;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index a9b145e30293c..57590d8d7da0e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -84,6 +84,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.namespace.LookupOptions;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
@@ -166,7 +167,7 @@ protected void cleanup() throws Exception {
         this.additionalPulsarTestContext.close();
     }
 
-    @BeforeMethod
+    @BeforeMethod(alwaysRun = true)
     protected void initializeState() throws PulsarAdminException {
         admin.namespaces().unload("public/default");
         reset(primaryLoadManager, secondaryLoadManager);
@@ -470,6 +471,33 @@ public Map filter(Map broker
         assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress());
     }
 
+    @Test
+    public void testStartOldLoadManager() throws Exception {
+        ServiceConfiguration defaultConf = getDefaultConf();
+        defaultConf.setAllowAutoTopicCreation(true);
+        defaultConf.setForceDeleteNamespaceAllowed(true);
+        defaultConf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+        defaultConf.setLoadBalancerSheddingEnabled(false);
+        try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) {
+            // start pulsar3 with old load manager
+            var pulsar3 = additionalPulsarTestContext.getPulsarService();
+
+            var availableBrokers = pulsar3.getLoadManager().get().getAvailableBrokers();
+            assertEquals(availableBrokers.size(), 1);
+            assertEquals(availableBrokers.iterator().next(), pulsar3.getLookupServiceAddress());
+
+            availableBrokers = pulsar1.getLoadManager().get().getAvailableBrokers();
+            assertEquals(availableBrokers.size(), 2);
+            assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress()));
+            assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress()));
+
+            availableBrokers = pulsar2.getLoadManager().get().getAvailableBrokers();
+            assertEquals(availableBrokers.size(), 2);
+            assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress()));
+            assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress()));
+        }
+    }
+
     @Test
     public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception {
         var topBundlesLoadDataStorePrimary =

From 1a6c28dd0072de05a544dbc9243bfbe6bccea5db Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 5 Apr 2023 18:10:55 -0500
Subject: [PATCH 143/404] [fix][broker] Only validate superuser access if authz
 enabled (#19989)

### Motivation

In #19455, I added a requirement that only the proxy role could supply an original principal. That check is only supposed to apply when the broker has authorization enabled. However, in one case, that was not the case. This PR does a check and returns early when authorization is not enabled in the broker.

See https://github.com/apache/pulsar/pull/19830#issuecomment-1492262201 for additional motivation.

### Modifications

* Update the `PulsarWebResource#validateSuperUserAccessAsync` to only validate when authentication and authorization are enabled in the configuration.

### Verifying this change

This is a trivial change. It'd be good to add tests, but I didn't include them here because this is a somewhat urgent fix. There was one test that broke because of this change, so there is at least some existing coverage.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/39
---
 .../pulsar/broker/web/PulsarWebResource.java  | 29 +++++++------------
 .../admin/v3/AdminApiTransactionTest.java     |  1 +
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
index c5713ebddaa2f..a182e4733fdfe 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
@@ -185,8 +185,8 @@ protected boolean hasSuperUserAccess() {
         return true;
     }
 
-    public CompletableFuture validateSuperUserAccessAsync(){
-        if (!config().isAuthenticationEnabled()) {
+    public CompletableFuture validateSuperUserAccessAsync() {
+        if (!config().isAuthenticationEnabled() || !config().isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(null);
         }
         String appId = clientAppId();
@@ -221,22 +221,15 @@ public CompletableFuture validateSuperUserAccessAsync(){
                         }
                     });
         } else {
-            if (config().isAuthorizationEnabled()) {
-                return pulsar.getBrokerService()
-                        .getAuthorizationService()
-                        .isSuperUser(appId, clientAuthData())
-                        .thenAccept(proxyAuthorizationSuccess -> {
-                            if (!proxyAuthorizationSuccess) {
-                                throw new RestException(Status.UNAUTHORIZED,
-                                        "This operation requires super-user access");
-                            }
-                        });
-            }
-            if (log.isDebugEnabled()) {
-                log.debug("Successfully authorized {} as super-user",
-                        appId);
-            }
-            return CompletableFuture.completedFuture(null);
+            return pulsar.getBrokerService()
+                    .getAuthorizationService()
+                    .isSuperUser(appId, clientAuthData())
+                    .thenAccept(proxyAuthorizationSuccess -> {
+                        if (!proxyAuthorizationSuccess) {
+                            throw new RestException(Status.UNAUTHORIZED,
+                                    "This operation requires super-user access");
+                        }
+                    });
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java
index 019b7c11fd579..0e51470da75a5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java
@@ -630,6 +630,7 @@ public void testUpdateTransactionCoordinatorNumber() throws Exception {
         Awaitility.await().until(() -> pulsar.getTransactionMetadataStoreService().getStores().size() ==
                         coordinatorSize * 2);
         pulsar.getConfiguration().setAuthenticationEnabled(true);
+        pulsar.getConfiguration().setAuthorizationEnabled(true);
         Set proxyRoles = spy(Set.class);
         doReturn(true).when(proxyRoles).contains(any());
         pulsar.getConfiguration().setProxyRoles(proxyRoles);

From 94ae340f94e004fb1de76a2d05e1e7160bb8eff1 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Thu, 6 Apr 2023 12:59:34 +0800
Subject: [PATCH 144/404] [fix][client] Release the orphan producers after the
 primary consumer is closed (#19858)

Motivation: The producers ["retryLetterProducer", "deadLetterProducer"] will be auto-created by consumers if enabled `DLQ`, but these producers will not close after consumers are closed.

Modifications: Auto close "retryLetterProducer" and "deadLetterProducer" after the primary consumer is closed
---
 .../pulsar/client/api/RetryTopicTest.java     | 48 +++++++++++++++++++
 .../pulsar/client/impl/ConsumerImpl.java      | 18 ++++++-
 2 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java
index 7012cb0d698ac..2ccae72143443 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
 import lombok.Data;
@@ -650,4 +651,51 @@ public void testRetryTopicException() throws Exception {
         consumer.close();
     }
 
+
+    @Test(timeOut = 30000L)
+    public void testRetryProducerWillCloseByConsumer() throws Exception {
+        final String topicName = "persistent://my-property/my-ns/tp_" + UUID.randomUUID().toString();
+        final String subscriptionName = "sub1";
+        final String topicRetry = topicName + "-" + subscriptionName + "-RETRY";
+        final String topicDLQ = topicName + "-" + subscriptionName + "-DLQ";
+
+        // Trigger the DLQ and retry topic creation.
+        final Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName(subscriptionName)
+                .subscriptionType(SubscriptionType.Shared)
+                .enableRetry(true)
+                .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(topicDLQ).maxRedeliverCount(2).build())
+                .receiverQueueSize(100)
+                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+                .subscribe();
+        final Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .create();
+        // send messages.
+        for (int i = 0; i < 5; i++) {
+            producer.newMessage()
+                    .value("msg-" + i)
+                    .sendAsync();
+        }
+        producer.flush();
+        for (int i = 0; i < 20; i++) {
+            Message msg = consumer.receive(5, TimeUnit.SECONDS);
+            if (msg != null) {
+                consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS);
+            } else {
+                break;
+            }
+        }
+
+        consumer.close();
+        producer.close();
+        admin.topics().delete(topicName, false);
+
+        // Verify: "retryLetterProducer" and "deadLetterProducer" will be closed by "consumer.close()", so these two
+        //  topics can be deleted successfully.
+        admin.topics().delete(topicRetry, false);
+        admin.topics().delete(topicDLQ, false);
+    }
+
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 74e6bf28baa1f..fb372566426d3 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -1041,7 +1041,23 @@ public CompletableFuture closeAsync() {
             });
         }
 
-        return closeFuture;
+        ArrayList> closeFutures = new ArrayList<>(4);
+        closeFutures.add(closeFuture);
+        if (retryLetterProducer != null) {
+            closeFutures.add(retryLetterProducer.closeAsync().whenComplete((ignore, ex) -> {
+                if (ex != null) {
+                    log.warn("Exception ignored in closing retryLetterProducer of consumer", ex);
+                }
+            }));
+        }
+        if (deadLetterProducer != null) {
+            closeFutures.add(deadLetterProducer.thenCompose(p -> p.closeAsync()).whenComplete((ignore, ex) -> {
+                if (ex != null) {
+                    log.warn("Exception ignored in closing deadLetterProducer of consumer", ex);
+                }
+            }));
+        }
+        return FutureUtil.waitForAll(closeFutures);
     }
 
     private void cleanupAtClose(CompletableFuture closeFuture, Throwable exception) {

From f76beda21134aff24cc0384e5447ffbe33c5c039 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 6 Apr 2023 00:59:21 -0500
Subject: [PATCH 145/404] [improve][txn] Cleanup how superusers abort txns
 (#19976)

### Motivation

This PR builds on https://github.com/apache/pulsar/pull/19467. When we modify/abort transactions, we need to make sure that authorization is checked for both the proxy and the client.

### Modifications

* Add a second authorization check when `originalPrincipal` is set in the `ServerCnx`.
* Fix a bug where we were not doing a deep copy of the `SubscriptionsList` object. (Tests caught this bug!)

### Verifying this change

Added a new test to cover some of the changes.

### Does this pull request potentially affect one of the following parts:

This is an internal change.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/38
---
 .../pulsar/broker/service/ServerCnx.java      | 49 ++++++++++++-------
 .../pulsar/broker/service/ServerCnxTest.java  | 43 ++++++++++++++++
 2 files changed, 73 insertions(+), 19 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 4fc79a124acd8..973dec8980ea0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -44,6 +44,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
@@ -2534,32 +2535,39 @@ protected void handleEndTxn(CommandEndTxn command) {
                 });
     }
 
-    private CompletableFuture verifyTxnOwnershipForTCToBrokerCommands() {
+    private CompletableFuture isSuperUser() {
+        assert ctx.executor().inEventLoop();
         if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) {
-            return getBrokerService()
-                    .getAuthorizationService()
-                    .isSuperUser(getPrincipal(), getAuthenticationData());
+            CompletableFuture isAuthRoleAuthorized = service.getAuthorizationService().isSuperUser(
+                    authRole, authenticationData);
+            if (originalPrincipal != null) {
+                CompletableFuture isOriginalPrincipalAuthorized = service.getAuthorizationService()
+                        .isSuperUser(originalPrincipal,
+                                originalAuthData != null ? originalAuthData : authenticationData);
+                return isOriginalPrincipalAuthorized.thenCombine(isAuthRoleAuthorized,
+                        (originalPrincipal, authRole) -> originalPrincipal && authRole);
+            } else {
+                return isAuthRoleAuthorized;
+            }
         } else {
             return CompletableFuture.completedFuture(true);
         }
     }
 
     private CompletableFuture verifyTxnOwnership(TxnID txnID) {
-        final String checkOwner = getPrincipal();
+        assert ctx.executor().inEventLoop();
         return service.pulsar().getTransactionMetadataStoreService()
-                .verifyTxnOwnership(txnID, checkOwner)
-                .thenCompose(isOwner -> {
+                .verifyTxnOwnership(txnID, getPrincipal())
+                .thenComposeAsync(isOwner -> {
                     if (isOwner) {
                         return CompletableFuture.completedFuture(true);
                     }
                     if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) {
-                        return getBrokerService()
-                                .getAuthorizationService()
-                                .isSuperUser(checkOwner, getAuthenticationData());
+                        return isSuperUser();
                     } else {
                         return CompletableFuture.completedFuture(false);
                     }
-                });
+                }, ctx.executor());
     }
 
     @Override
@@ -2576,10 +2584,10 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) {
                     txnID, txnAction);
         }
         CompletableFuture> topicFuture = service.getTopicIfExists(TopicName.get(topic).toString());
-        topicFuture.thenAccept(optionalTopic -> {
+        topicFuture.thenAcceptAsync(optionalTopic -> {
             if (optionalTopic.isPresent()) {
-                // we only accept super user becase this endpoint is reserved for tc to broker communication
-                verifyTxnOwnershipForTCToBrokerCommands()
+                // we only accept superuser because this endpoint is reserved for tc to broker communication
+                isSuperUser()
                         .thenCompose(isOwner -> {
                             if (!isOwner) {
                                 return failedFutureTxnTcNotAllowed(txnID);
@@ -2629,7 +2637,7 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) {
                             return null;
                 });
             }
-        }).exceptionally(e -> {
+        }, ctx.executor()).exceptionally(e -> {
             log.error("handleEndTxnOnPartition fail ! topic {}, "
                             + "txnId: [{}], txnAction: [{}]", topic, txnID,
                     TxnAction.valueOf(txnAction), e.getCause());
@@ -2658,7 +2666,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) {
         }
 
         CompletableFuture> topicFuture = service.getTopicIfExists(TopicName.get(topic).toString());
-        topicFuture.thenAccept(optionalTopic -> {
+        topicFuture.thenAcceptAsync(optionalTopic -> {
             if (optionalTopic.isPresent()) {
                 Subscription subscription = optionalTopic.get().getSubscription(subName);
                 if (subscription == null) {
@@ -2670,7 +2678,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) {
                     return;
                 }
                 // we only accept super user becase this endpoint is reserved for tc to broker communication
-                verifyTxnOwnershipForTCToBrokerCommands()
+                isSuperUser()
                         .thenCompose(isOwner -> {
                             if (!isOwner) {
                                 return failedFutureTxnTcNotAllowed(txnID);
@@ -2720,7 +2728,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) {
                             return null;
                 });
             }
-        }).exceptionally(e -> {
+        }, ctx.executor()).exceptionally(e -> {
             log.error("handleEndTxnOnSubscription fail ! topic: {}, subscription: {}"
                             + "txnId: [{}], txnAction: [{}]", topic, subName,
                     txnID, TxnAction.valueOf(txnAction), e.getCause());
@@ -2757,7 +2765,10 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) {
         checkArgument(state == State.Connected);
         final TxnID txnID = new TxnID(command.getTxnidMostBits(), command.getTxnidLeastBits());
         final long requestId = command.getRequestId();
-        final List subscriptionsList = command.getSubscriptionsList();
+        final List subscriptionsList = new ArrayList<>();
+        for (org.apache.pulsar.common.api.proto.Subscription sub : command.getSubscriptionsList()) {
+            subscriptionsList.add(new org.apache.pulsar.common.api.proto.Subscription().copyFrom(sub));
+        }
         if (log.isDebugEnabled()) {
             log.debug("Receive add published partition to txn request {} from {} with txnId {}",
                     requestId, remoteAddress, txnID);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 4580f028de2b0..874d43896223c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -33,6 +33,7 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.matches;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -56,6 +57,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -75,6 +77,7 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.auth.MockAlwaysExpiredAuthenticationProvider;
+import org.apache.pulsar.broker.auth.MockAuthorizationProvider;
 import org.apache.pulsar.broker.auth.MockMutableAuthenticationProvider;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
@@ -3171,6 +3174,46 @@ public void sendAddPartitionToTxnResponseFailed() throws Exception {
         channel.finish();
     }
 
+    @Test(timeOut = 30000)
+    public void sendAddPartitionToTxnResponseFailedAuth() throws Exception {
+        AuthenticationService authenticationService = mock(AuthenticationService.class);
+        AuthenticationProvider authenticationProvider = new MockAuthenticationProvider();
+        String authMethodName = authenticationProvider.getAuthMethodName();
+        when(brokerService.getAuthenticationService()).thenReturn(authenticationService);
+        when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider);
+        svcConfig.setAuthenticationEnabled(true);
+        svcConfig.setProxyRoles(Set.of("pass.fail"));
+
+        svcConfig.setAuthorizationProvider(MockAuthorizationProvider.class.getName());
+        AuthorizationService authorizationService =
+                spyWithClassAndConstructorArgsRecordingInvocations(AuthorizationService.class, svcConfig,
+                        pulsarTestContext.getPulsarResources());
+        when(brokerService.getAuthorizationService()).thenReturn(authorizationService);
+        svcConfig.setAuthorizationEnabled(true);
+
+        final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class);
+        when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(false));
+        when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore);
+        svcConfig.setTransactionCoordinatorEnabled(true);
+        resetChannel();
+
+        ByteBuf connect = Commands.newConnect(authMethodName, "pass.fail", "test", "localhost",
+                "pass.pass", "pass.pass", authMethodName);
+        channel.writeInbound(connect);
+        Object connectResponse = getResponse();
+        assertTrue(connectResponse instanceof CommandConnected);
+
+        ByteBuf clientCommand = Commands.newAddPartitionToTxn(89L, 1L, 12L,
+                List.of("tenant/ns/topic1"));
+        channel.writeInbound(clientCommand);
+        CommandAddPartitionToTxnResponse response = (CommandAddPartitionToTxnResponse) getResponse();
+
+        assertEquals(response.getError(), ServerError.TransactionNotFound);
+        verify(txnStore, never()).addProducedPartitionToTxn(any(TxnID.class), any());
+
+        channel.finish();
+    }
+
     @Test(timeOut = 30000)
     public void sendAddSubscriptionToTxnResponse() throws Exception {
         final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class);

From dd054080490f01e9ba9694756b335e3e10a637d5 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 6 Apr 2023 13:00:49 +0300
Subject: [PATCH 146/404] [fix][proxy] Fix connection read timeout handling in
 Pulsar Proxy (#20014)

---
 .../proxy/server/DirectProxyHandler.java      |  6 ++-
 .../pulsar/proxy/server/ProxyConnection.java  |  3 ++
 .../proxy/server/ProxyReadTimeoutHandler.java | 46 -------------------
 .../server/ServiceChannelInitializer.java     |  3 +-
 4 files changed, 10 insertions(+), 48 deletions(-)
 delete mode 100644 pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java
index 23c7faa2d4bb7..cb93f885a1d68 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java
@@ -43,6 +43,7 @@
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.timeout.ReadTimeoutHandler;
 import io.netty.util.CharsetUtil;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
@@ -205,7 +206,7 @@ protected void initChannel(SocketChannel ch) {
                 int brokerProxyReadTimeoutMs = service.getConfiguration().getBrokerProxyReadTimeoutMs();
                 if (brokerProxyReadTimeoutMs > 0) {
                     ch.pipeline().addLast("readTimeoutHandler",
-                            new ProxyReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS));
+                            new ReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS));
                 }
                 ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder(
                         service.getConfiguration().getMaxMessageSize() + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0,
@@ -362,6 +363,9 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce
 
                 if (service.proxyZeroCopyModeEnabled && service.proxyLogLevel == 0) {
                     if (!isTlsOutboundChannel && !DirectProxyHandler.this.proxyConnection.isTlsInboundChannel) {
+                        if (ctx.pipeline().get("readTimeoutHandler") != null) {
+                            ctx.pipeline().remove("readTimeoutHandler");
+                        }
                         ProxyConnection.spliceNIC2NIC((EpollSocketChannel) ctx.channel(),
                                         (EpollSocketChannel) inboundChannel, ProxyConnection.SPLICE_BYTES)
                                 .addListener(future -> {
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index b7f5534ee81fb..9530389b524b3 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -263,6 +263,9 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) throws Exce
 
                 if (service.proxyZeroCopyModeEnabled && service.proxyLogLevel == 0) {
                     if (!directProxyHandler.isTlsOutboundChannel && !isTlsInboundChannel) {
+                        if (ctx.pipeline().get("readTimeoutHandler") != null) {
+                            ctx.pipeline().remove("readTimeoutHandler");
+                        }
                         spliceNIC2NIC((EpollSocketChannel) ctx.channel(),
                                 (EpollSocketChannel) directProxyHandler.outboundChannel, SPLICE_BYTES)
                                 .addListener(future -> {
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java
deleted file mode 100644
index df650a0ca16c1..0000000000000
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyReadTimeoutHandler.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.proxy.server;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.timeout.IdleStateHandler;
-import io.netty.handler.timeout.ReadTimeoutHandler;
-import java.lang.reflect.Field;
-import java.util.concurrent.TimeUnit;
-
-public class ProxyReadTimeoutHandler extends ReadTimeoutHandler {
-
-    private final Field readingField;
-
-    public ProxyReadTimeoutHandler(long timeout, TimeUnit unit) {
-        super(timeout, unit);
-        try {
-            this.readingField = IdleStateHandler.class.getDeclaredField("reading");
-            this.readingField.setAccessible(true);
-        } catch (NoSuchFieldException e) {
-            throw new IllegalArgumentException("Exception caused while get 'reading' field", e);
-        }
-    }
-
-    @Override
-    public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
-        this.readingField.setBoolean(this, true);
-        super.channelReadComplete(ctx);
-    }
-}
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java
index 7e7ba31b79fe3..19f4002ad52ce 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java
@@ -25,6 +25,7 @@
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.timeout.ReadTimeoutHandler;
 import java.util.concurrent.TimeUnit;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder;
@@ -106,7 +107,7 @@ protected void initChannel(SocketChannel ch) throws Exception {
         }
         if (brokerProxyReadTimeoutMs > 0) {
             ch.pipeline().addLast("readTimeoutHandler",
-                    new ProxyReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS));
+                    new ReadTimeoutHandler(brokerProxyReadTimeoutMs, TimeUnit.MILLISECONDS));
         }
         if (proxyService.getConfiguration().isHaProxyProtocolEnabled()) {
             ch.pipeline().addLast(OptionalProxyProtocolDecoder.NAME, new OptionalProxyProtocolDecoder());

From 067e3c038edf406b27f8f91847f688f559a2ba25 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 6 Apr 2023 08:45:58 -0500
Subject: [PATCH 147/404] [feat][fn] PIP-257: Support mounting k8s
 ServiceAccount for OIDC auth (#19888)

PIP: #19771

### Motivation

In order to make OIDC work with functions, we must give them a way to authenticate with the broker using tokens that are able to be validated by an using an Authorization Server. This PR introduces the `KubernetesServiceAccountAuthProvider`.

### Modifications

* Create an `KubernetesServiceAccountAuthProvider` implementation. It adds a service account token volume projection as defined in the k8s docs [here](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-token-volume-projection). The implementation provides a way to specify the expiration time that the token will receive.

* Instead of creating a secret with the broker's trusted `ca.crt` in it, this new `KubernetesServiceAccountAuthProvider` expects a secret to already exist with the `ca.crt`. The major advantage for this implementation is that when the `ca.crt` is rotated, we can refresh it (assuming the client is configured to observe the updated file).

* Add support for specifying the token's expiration and audience.

* One point of divergence from the `KubernetesSecretsTokenAuthProvider` implementation is that I did not provide a way for functions to authenticate as the anonymous role. It seems like a stretch that functions would use such authentication because it will not be multi-tenant. However, if that is a concern, we can add the support.

* The feature will be configured with the following yaml:

```yaml
functionRuntimeFactoryConfigs:
  kubernetesFunctionAuthProviderConfig:
    brokerClientTrustCertsSecretName: "secret-name"
    serviceAccountTokenExpirationSeconds: 3600
    serviceAccountTokenAudience: "pulsar-cluster-audience"
```

### Verifying this change

I verified the correctness of the code with unit tests. I'll verify the integration with k8s once we've determined this PR's design is correct.

### Does this pull request potentially affect one of the following parts:

This adds new configuration options to the function worker.

### Documentation

- [x] `doc-required`

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/36
---
 .../auth/KubernetesFunctionAuthProvider.java  |  12 +
 ...rnetesServiceAccountTokenAuthProvider.java | 208 ++++++++++++++++++
 .../kubernetes/KubernetesRuntimeFactory.java  |   2 +-
 .../KubernetesRuntimeFactoryConfig.java       |   6 +
 ...esServiceAccountTokenAuthProviderTest.java | 128 +++++++++++
 5 files changed, 355 insertions(+), 1 deletion(-)
 create mode 100644 pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java
 create mode 100644 pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java
index 29365424f547f..3d05d519552c9 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesFunctionAuthProvider.java
@@ -20,6 +20,7 @@
 
 import io.kubernetes.client.openapi.apis.CoreV1Api;
 import io.kubernetes.client.openapi.models.V1StatefulSet;
+import java.util.Map;
 import java.util.Optional;
 import org.apache.pulsar.common.util.Reflections;
 import org.apache.pulsar.functions.proto.Function;
@@ -31,6 +32,11 @@ public interface KubernetesFunctionAuthProvider extends FunctionAuthProvider {
 
     void initialize(CoreV1Api coreClient);
 
+    /**
+     * @deprecated use
+     * {@link #initialize(CoreV1Api, byte[], java.util.function.Function, Map)}
+     */
+    @Deprecated(since = "3.0.0")
     default void initialize(CoreV1Api coreClient, byte[] caBytes,
                             java.util.function.Function namespaceCustomizerFunc) {
         setCaBytes(caBytes);
@@ -38,6 +44,12 @@ default void initialize(CoreV1Api coreClient, byte[] caBytes,
         initialize(coreClient);
     }
 
+    default void initialize(CoreV1Api coreClient, byte[] caBytes,
+                            java.util.function.Function namespaceCustomizerFunc,
+                            Map config) {
+        initialize(coreClient, caBytes, namespaceCustomizerFunc);
+    }
+
     default void setCaBytes(byte[] caBytes) {
 
     }
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java
new file mode 100644
index 0000000000000..06fadec42d42f
--- /dev/null
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProvider.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.auth;
+
+import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1KeyToPath;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1ProjectedVolumeSource;
+import io.kubernetes.client.openapi.models.V1SecretVolumeSource;
+import io.kubernetes.client.openapi.models.V1ServiceAccountTokenProjection;
+import io.kubernetes.client.openapi.models.V1StatefulSet;
+import io.kubernetes.client.openapi.models.V1Volume;
+import io.kubernetes.client.openapi.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1VolumeProjection;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.functions.instance.AuthenticationConfig;
+import org.apache.pulsar.functions.proto.Function;
+import org.eclipse.jetty.util.StringUtil;
+
+/**
+ * Kubernetes Function Authentication Provider that adds Service Account Token Projection to a function pod's container
+ * definition. This token can be used to authenticate the function instance with the broker and the function worker via
+ * OpenId Connect when each server is configured to trust the kubernetes issuer. See docs for additional details.
+ * Relevant settings:
+ * 

+ *     brokerClientTrustCertsSecretName: The Kubernetes secret containing the broker's trust certs. If it is not set,
+ *     the function will not use a custom trust store. The secret must already exist in each function's target
+ *     namespace. The secret must contain a key named `ca.crt` with the trust certs. Only the ca.crt will be mounted.
+ * 

+ * 

+ *     serviceAccountTokenExpirationSeconds: The expiration for the token created by the
+ *     {@link KubernetesServiceAccountTokenAuthProvider}. The default value is 3600 seconds.
+ * 

+ * 

+ *     serviceAccountTokenAudience: The audience for the token created by the
+ *     {@link KubernetesServiceAccountTokenAuthProvider}.
+ * 

+ * Note: the pod inherits the namespace's default service account.
+ */
+public class KubernetesServiceAccountTokenAuthProvider implements KubernetesFunctionAuthProvider {
+
+    private static final String BROKER_CLIENT_TRUST_CERTS_SECRET_NAME = "brokerClientTrustCertsSecretName";
+    private static final String SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS = "serviceAccountTokenExpirationSeconds";
+    private static final String SERVICE_ACCOUNT_TOKEN_AUDIENCE = "serviceAccountTokenAudience";
+
+    private static final String SERVICE_ACCOUNT_VOLUME_NAME = "service-account-token";
+    private static final String TRUST_CERT_VOLUME_NAME = "ca-cert";
+    private static final String DEFAULT_MOUNT_DIR = "/etc/auth";
+    private static final String FUNCTION_AUTH_TOKEN = "token";
+    private static final String FUNCTION_CA_CERT = "ca.crt";
+    private static final String DEFAULT_CERT_PATH = DEFAULT_MOUNT_DIR + "/" + FUNCTION_CA_CERT;
+    private String brokerTrustCertsSecretName;
+    private long serviceAccountTokenExpirationSeconds;
+    private String serviceAccountTokenAudience;
+
+    @Override
+    public void initialize(CoreV1Api coreClient, byte[] caBytes,
+                           java.util.function.Function namespaceCustomizerFunc,
+                           Map config) {
+        setNamespaceProviderFunc(namespaceCustomizerFunc);
+        Object certSecretName = config.get(BROKER_CLIENT_TRUST_CERTS_SECRET_NAME);
+        if (certSecretName instanceof String) {
+            brokerTrustCertsSecretName = (String) certSecretName;
+        } else if (certSecretName != null) {
+            // Throw exception because user set this configuration, but it isn't valid.
+            throw new IllegalArgumentException("Invalid value for " + BROKER_CLIENT_TRUST_CERTS_SECRET_NAME
+                    + ". Expected a string.");
+        }
+        Object tokenExpirationSeconds = config.get(SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS);
+        if (tokenExpirationSeconds instanceof Long) {
+            serviceAccountTokenExpirationSeconds = (Long) tokenExpirationSeconds;
+        } else if (tokenExpirationSeconds instanceof String) {
+            try {
+                serviceAccountTokenExpirationSeconds = Long.parseLong((String) tokenExpirationSeconds);
+            } catch (NumberFormatException e) {
+                throw new IllegalArgumentException("Invalid value for " + SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS
+                        + ". Expected a long.");
+            }
+        } else if (tokenExpirationSeconds != null) {
+            // Throw exception because user set this configuration, but it isn't valid.
+            throw new IllegalArgumentException("Invalid value for " + SERVICE_ACCOUNT_TOKEN_EXPIRATION_SECONDS
+                    + ". Expected a long.");
+        }
+        Object tokenAudience = config.get(SERVICE_ACCOUNT_TOKEN_AUDIENCE);
+        if (tokenAudience instanceof String) {
+            serviceAccountTokenAudience = (String) tokenAudience;
+        } else if (tokenAudience != null) {
+            throw new IllegalArgumentException("Invalid value for " + SERVICE_ACCOUNT_TOKEN_AUDIENCE
+                    + ". Expected a string.");
+        }
+    }
+
+    @Override
+    public void configureAuthenticationConfig(AuthenticationConfig authConfig,
+                                              Optional functionAuthData) {
+        authConfig.setClientAuthenticationPlugin(AuthenticationToken.class.getName());
+        authConfig.setClientAuthenticationParameters(Paths.get(DEFAULT_MOUNT_DIR, FUNCTION_AUTH_TOKEN)
+                .toUri().toString());
+        if (StringUtil.isNotBlank(brokerTrustCertsSecretName)) {
+            authConfig.setTlsTrustCertsFilePath(DEFAULT_CERT_PATH);
+        }
+    }
+
+    /**
+     * No need to cache anything. Kubernetes generates the token used for authentication.
+     */
+    @Override
+    public Optional cacheAuthData(Function.FunctionDetails funcDetails,
+                                                    AuthenticationDataSource authenticationDataSource)
+            throws Exception {
+        return Optional.empty();
+    }
+
+    /**
+     * No need to update anything. Kubernetes updates the token used for authentication.
+     */
+    @Override
+    public Optional updateAuthData(Function.FunctionDetails funcDetails,
+                                                     Optional existingFunctionAuthData,
+                                                     AuthenticationDataSource authenticationDataSource)
+            throws Exception {
+        return Optional.empty();
+    }
+
+    /**
+     * No need to clean up anything. Kubernetes cleans up the secret when the pod is deleted.
+     */
+    @Override
+    public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional functionAuthData)
+            throws Exception {
+
+    }
+
+    @Override
+    public void initialize(CoreV1Api coreClient) {
+    }
+
+    @Override
+    public void configureAuthDataStatefulSet(V1StatefulSet statefulSet, Optional functionAuthData) {
+        V1PodSpec podSpec = statefulSet.getSpec().getTemplate().getSpec();
+        // configure pod mount secret with auth token
+        if (StringUtil.isNotBlank(brokerTrustCertsSecretName)) {
+            podSpec.addVolumesItem(createTrustCertVolume());
+        }
+        podSpec.addVolumesItem(createServiceAccountVolume());
+        podSpec.getContainers().forEach(this::addVolumeMountsToContainer);
+    }
+
+    private V1Volume createServiceAccountVolume() {
+        V1ProjectedVolumeSource projectedVolumeSource = new V1ProjectedVolumeSource();
+        V1VolumeProjection volumeProjection = new V1VolumeProjection();
+        volumeProjection.serviceAccountToken(
+                new V1ServiceAccountTokenProjection()
+                        .audience(serviceAccountTokenAudience)
+                        .expirationSeconds(serviceAccountTokenExpirationSeconds)
+                        .path(FUNCTION_AUTH_TOKEN));
+        projectedVolumeSource.addSourcesItem(volumeProjection);
+        return new V1Volume()
+                .name(SERVICE_ACCOUNT_VOLUME_NAME)
+                .projected(projectedVolumeSource);
+    }
+
+    private V1Volume createTrustCertVolume() {
+        return new V1Volume()
+                .name(TRUST_CERT_VOLUME_NAME)
+                .secret(new V1SecretVolumeSource()
+                        .secretName(brokerTrustCertsSecretName)
+                        .addItemsItem(new V1KeyToPath()
+                                .key(FUNCTION_CA_CERT)
+                                .path(FUNCTION_CA_CERT)));
+    }
+
+    private void addVolumeMountsToContainer(V1Container container) {
+        container.addVolumeMountsItem(
+                new V1VolumeMount()
+                        .name(SERVICE_ACCOUNT_VOLUME_NAME)
+                        .mountPath(DEFAULT_MOUNT_DIR)
+                        .readOnly(true));
+        if (StringUtil.isNotBlank(brokerTrustCertsSecretName)) {
+            container.addVolumeMountsItem(
+                    new V1VolumeMount()
+                            .name(TRUST_CERT_VOLUME_NAME)
+                            .mountPath(DEFAULT_MOUNT_DIR)
+                            .readOnly(true));
+        }
+    }
+}
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
index 692884a303d58..895304138a530 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
@@ -252,7 +252,7 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic
                 kubernetesFunctionAuthProvider.initialize(coreClient, serverCaBytes,
                         (funcDetails) -> getRuntimeCustomizer()
                                 .map((customizer) -> customizer.customizeNamespace(funcDetails, jobNamespace))
-                                .orElse(jobNamespace));
+                                .orElse(jobNamespace), factoryConfig.getKubernetesFunctionAuthProviderConfig());
                 this.authProvider = Optional.of(kubernetesFunctionAuthProvider);
             }
         } else {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java
index 5cbca7b65bfd7..43cdc035076c3 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.functions.runtime.kubernetes;
 
+import java.util.HashMap;
 import java.util.Map;
 import lombok.Data;
 import lombok.experimental.Accessors;
@@ -169,4 +170,9 @@ public class KubernetesRuntimeFactoryConfig {
     )
     protected int gracePeriodSeconds = 5;
 
+    @FieldContext(
+            doc = "A map of custom configurations passed to implementations of the KubernetesFunctionAuthProvider"
+                    + " interface."
+    )
+    private Map kubernetesFunctionAuthProviderConfig = new HashMap<>();
 }
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java
new file mode 100644
index 0000000000000..2e8cf75151044
--- /dev/null
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesServiceAccountTokenAuthProviderTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.auth;
+
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
+import io.kubernetes.client.openapi.models.V1ServiceAccountTokenProjection;
+import io.kubernetes.client.openapi.models.V1StatefulSet;
+import io.kubernetes.client.openapi.models.V1StatefulSetSpec;
+import io.kubernetes.client.openapi.models.V1Volume;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.functions.instance.AuthenticationConfig;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class KubernetesServiceAccountTokenAuthProviderTest {
+    @Test
+    public void testConfigureAuthDataStatefulSet() {
+        HashMap config = new HashMap<>();
+        config.put("brokerClientTrustCertsSecretName", "my-secret");
+        config.put("serviceAccountTokenExpirationSeconds", "600");
+        config.put("serviceAccountTokenAudience", "my-audience");
+        KubernetesServiceAccountTokenAuthProvider provider = new KubernetesServiceAccountTokenAuthProvider();
+        provider.initialize(null, null, (fd) -> "default", config);
+
+        // Create a stateful set with a container
+        V1StatefulSet statefulSet = new V1StatefulSet();
+        statefulSet.setSpec(
+                new V1StatefulSetSpec().template(
+                        new V1PodTemplateSpec().spec(
+                                new V1PodSpec().containers(
+                                        Collections.singletonList(new V1Container())))));
+        provider.configureAuthDataStatefulSet(statefulSet, Optional.empty());
+
+        List volumes = statefulSet.getSpec().getTemplate().getSpec().getVolumes();
+        Assert.assertEquals(volumes.size(), 2);
+
+        Assert.assertEquals(volumes.get(0).getName(), "ca-cert");
+        Assert.assertEquals(volumes.get(0).getSecret().getSecretName(), "my-secret");
+        Assert.assertEquals(volumes.get(0).getSecret().getItems().size(), 1);
+        Assert.assertEquals(volumes.get(0).getSecret().getItems().get(0).getKey(), "ca.crt");
+        Assert.assertEquals(volumes.get(0).getSecret().getItems().get(0).getPath(), "ca.crt");
+
+
+        Assert.assertEquals(volumes.get(1).getName(), "service-account-token");
+        Assert.assertEquals(volumes.get(1).getProjected().getSources().size(), 1);
+        V1ServiceAccountTokenProjection tokenProjection =
+                volumes.get(1).getProjected().getSources().get(0).getServiceAccountToken();
+        Assert.assertEquals(tokenProjection.getExpirationSeconds(), 600);
+        Assert.assertEquals(tokenProjection.getAudience(), "my-audience");
+
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().size(), 1);
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().size(), 2);
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getName(), "service-account-token");
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getMountPath(), "/etc/auth");
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(1).getName(), "ca-cert");
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(1).getMountPath(), "/etc/auth");
+    }
+
+    @Test
+    public void testConfigureAuthDataStatefulSetNoCa() {
+
+        HashMap config = new HashMap<>();
+        config.put("serviceAccountTokenExpirationSeconds", "600");
+        config.put("serviceAccountTokenAudience", "pulsar-cluster");
+        KubernetesServiceAccountTokenAuthProvider provider = new KubernetesServiceAccountTokenAuthProvider();
+        provider.initialize(null, null, (fd) -> "default", config);
+
+        // Create a stateful set with a container
+        V1StatefulSet statefulSet = new V1StatefulSet();
+        statefulSet.setSpec(
+                new V1StatefulSetSpec().template(
+                        new V1PodTemplateSpec().spec(
+                                new V1PodSpec().containers(
+                                        Collections.singletonList(new V1Container())))));
+        provider.configureAuthDataStatefulSet(statefulSet, Optional.empty());
+
+        List volumes = statefulSet.getSpec().getTemplate().getSpec().getVolumes();
+        Assert.assertEquals(volumes.size(), 1);
+
+        Assert.assertEquals(volumes.get(0).getName(), "service-account-token");
+        Assert.assertEquals(volumes.get(0).getProjected().getSources().size(), 1);
+        V1ServiceAccountTokenProjection tokenProjection =
+                volumes.get(0).getProjected().getSources().get(0).getServiceAccountToken();
+        Assert.assertEquals(tokenProjection.getExpirationSeconds(), 600);
+        Assert.assertEquals(tokenProjection.getAudience(), "pulsar-cluster");
+
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().size(), 1);
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().size(), 1);
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getName(), "service-account-token");
+        Assert.assertEquals(statefulSet.getSpec().getTemplate().getSpec().getContainers().get(0).getVolumeMounts().get(0).getMountPath(), "/etc/auth");
+    }
+
+    @Test
+    public void configureAuthenticationConfig() {
+        HashMap config = new HashMap<>();
+        config.put("brokerClientTrustCertsSecretName", "my-secret");
+        KubernetesServiceAccountTokenAuthProvider provider = new KubernetesServiceAccountTokenAuthProvider();
+        provider.initialize(null, null, (fd) -> "default", config);
+        AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build();
+        provider.configureAuthenticationConfig(authenticationConfig, Optional.empty());
+
+        Assert.assertEquals(authenticationConfig.getClientAuthenticationPlugin(), AuthenticationToken.class.getName());
+        Assert.assertEquals(authenticationConfig.getClientAuthenticationParameters(), "file:///etc/auth/token");
+        Assert.assertEquals(authenticationConfig.getTlsTrustCertsFilePath(), "/etc/auth/ca.crt");
+    }
+}

From 42a6969efc8fdc16f2d883d3c4f2865bdb460091 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Thu, 6 Apr 2023 21:50:45 +0800
Subject: [PATCH 148/404] [fix][broker] Return if
 AbstractDispatcherSingleActiveConsumer closed (#19934)

---
 ...AbstractDispatcherSingleActiveConsumer.java |  1 +
 ...nPersistentDispatcherMultipleConsumers.java |  2 +-
 ...ntStickyKeyDispatcherMultipleConsumers.java |  9 +++++++++
 ...ntStickyKeyDispatcherMultipleConsumers.java | 11 +++++++++++
 ...rsistentDispatcherFailoverConsumerTest.java | 14 ++++++++++++++
 ...ickyKeyDispatcherMultipleConsumersTest.java | 18 +++++++++++++++---
 ...ickyKeyDispatcherMultipleConsumersTest.java | 11 +++++++++++
 7 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java
index 2e22a80250cc3..1b6df5f809327 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java
@@ -161,6 +161,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce
         if (IS_CLOSED_UPDATER.get(this) == TRUE) {
             log.warn("[{}] Dispatcher is already closed. Closing consumer {}", this.topicName, consumer);
             consumer.disconnect();
+            return;
         }
 
         if (subscriptionType == SubType.Exclusive && !consumers.isEmpty()) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java
index f76fc09ba2ebe..d679106094178 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java
@@ -47,7 +47,7 @@ public class NonPersistentDispatcherMultipleConsumers extends AbstractDispatcher
     protected final Subscription subscription;
 
     private CompletableFuture closeFuture = null;
-    private final String name;
+    protected final String name;
     protected final Rate msgDrop;
     protected static final AtomicIntegerFieldUpdater
             TOTAL_AVAILABLE_PERMITS_UPDATER = AtomicIntegerFieldUpdater
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java
index f26e1399be71d..23814bfdbe22e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java
@@ -39,6 +39,8 @@
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
 import org.apache.pulsar.common.api.proto.KeySharedMode;
 import org.apache.pulsar.common.protocol.Commands;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class NonPersistentStickyKeyDispatcherMultipleConsumers extends NonPersistentDispatcherMultipleConsumers {
 
@@ -84,6 +86,11 @@ public NonPersistentStickyKeyDispatcherMultipleConsumers(NonPersistentTopic topi
 
     @Override
     public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+        if (IS_CLOSED_UPDATER.get(this) == TRUE) {
+            log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer);
+            consumer.disconnect();
+            return;
+        }
         super.addConsumer(consumer);
         try {
             selector.addConsumer(consumer);
@@ -168,4 +175,6 @@ public KeySharedMode getKeySharedMode() {
     public boolean hasSameKeySharedPolicy(KeySharedMeta ksm) {
         return (ksm.getKeySharedMode() == this.keySharedMode);
     }
+
+    private static final Logger log = LoggerFactory.getLogger(NonPersistentStickyKeyDispatcherMultipleConsumers.class);
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index 6180f29c96d1d..0c29421edd4ef 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.service.persistent;
 
 import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
+import com.google.common.annotations.VisibleForTesting;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -100,8 +101,18 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
         }
     }
 
+    @VisibleForTesting
+    public StickyKeyConsumerSelector getSelector() {
+        return selector;
+    }
+
     @Override
     public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+        if (IS_CLOSED_UPDATER.get(this) == TRUE) {
+            log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer);
+            consumer.disconnect();
+            return;
+        }
         super.addConsumer(consumer);
         try {
             selector.addConsumer(consumer);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java
index 0dafe4bca3e2c..631b702dfbd08 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java
@@ -233,6 +233,20 @@ private void verifyActiveConsumerChange(CommandActiveConsumerChange change,
         assertEquals(isActive, change.isIsActive());
     }
 
+    @Test(timeOut = 10000)
+    public void testAddConsumerWhenClosed() throws Exception {
+        PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, pulsarTestContext.getBrokerService());
+        PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false);
+        PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock,
+                SubType.Failover, 0, topic, sub);
+        pdfc.close().get();
+
+        Consumer consumer = mock(Consumer.class);
+        pdfc.addConsumer(consumer);
+        verify(consumer, times(1)).disconnect();
+        assertEquals(0, pdfc.consumers.size());
+    }
+
     @Test
     public void testConsumerGroupChangesWithOldNewConsumers() throws Exception {
         PersistentTopic topic =
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java
index 64807c73852e3..b2638d53ab1c3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -31,6 +31,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
@@ -47,6 +48,7 @@
 import org.apache.pulsar.broker.service.EntryBatchSizes;
 import org.apache.pulsar.broker.service.HashRangeAutoSplitStickyKeyConsumerSelector;
 import org.apache.pulsar.broker.service.RedeliveryTracker;
+import org.apache.pulsar.broker.service.StickyKeyConsumerSelector;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies;
 import org.apache.pulsar.common.protocol.Commands;
@@ -62,6 +64,7 @@ public class NonPersistentStickyKeyDispatcherMultipleConsumersTest {
     private ServiceConfiguration configMock;
 
     private NonPersistentStickyKeyDispatcherMultipleConsumers nonpersistentDispatcher;
+    private StickyKeyConsumerSelector selector;
 
     final String topicName = "non-persistent://public/default/testTopic";
 
@@ -88,10 +91,19 @@ public void setup() throws Exception {
         doReturn(topicPolicies).when(topicMock).getHierarchyTopicPolicies();
 
         subscriptionMock = mock(NonPersistentSubscription.class);
-
+        selector = new HashRangeAutoSplitStickyKeyConsumerSelector();
         nonpersistentDispatcher = new NonPersistentStickyKeyDispatcherMultipleConsumers(
-            topicMock, subscriptionMock,
-            new HashRangeAutoSplitStickyKeyConsumerSelector());
+            topicMock, subscriptionMock, selector);
+    }
+
+    @Test(timeOut = 10000)
+    public void testAddConsumerWhenClosed() throws Exception {
+        nonpersistentDispatcher.close().get();
+        Consumer consumer = mock(Consumer.class);
+        nonpersistentDispatcher.addConsumer(consumer);
+        verify(consumer, times(1)).disconnect();
+        assertEquals(0, nonpersistentDispatcher.getConsumers().size());
+        assertTrue(selector.getConsumerKeyHashRanges().isEmpty());
     }
 
     @Test(timeOut = 10000)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
index 1c23afd957a64..48a4bfc923608 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -35,6 +35,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
@@ -159,6 +160,16 @@ public void cleanup() {
         }
     }
 
+    @Test(timeOut = 10000)
+    public void testAddConsumerWhenClosed() throws Exception {
+        persistentDispatcher.close().get();
+        Consumer consumer = mock(Consumer.class);
+        persistentDispatcher.addConsumer(consumer);
+        verify(consumer, times(1)).disconnect();
+        assertEquals(0, persistentDispatcher.getConsumers().size());
+        assertTrue(persistentDispatcher.getSelector().getConsumerKeyHashRanges().isEmpty());
+    }
+
     @Test
     public void testSendMarkerMessage() {
         try {

From 8b0a0a388ad9b15defcb487fbd9ac153d7a06888 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Fri, 7 Apr 2023 13:50:44 +0800
Subject: [PATCH 149/404] [improve][ml] Add Read cache misses metric for ledger
 (#20001)

---
 .../mledger/ManagedLedgerMXBean.java          |   5 +
 .../mledger/impl/ManagedLedgerMBeanImpl.java  |  11 ++
 .../impl/cache/EntryCacheDisabled.java        |   2 +
 .../impl/cache/RangeEntryCacheImpl.java       |   2 +
 .../stats/metrics/ManagedLedgerMetrics.java   |   2 +
 .../prometheus/AggregatedBrokerStats.java     |   3 +
 .../prometheus/AggregatedNamespaceStats.java  |   1 +
 .../stats/prometheus/ManagedLedgerStats.java  |   2 +
 .../prometheus/NamespaceStatsAggregator.java  |   5 +
 .../broker/stats/prometheus/TopicStats.java   |   3 +
 .../broker/stats/PrometheusMetricsTest.java   | 118 ++++++++++++++++++
 11 files changed, 154 insertions(+)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
index 94c2f61e00afe..50a3ffb157961 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
@@ -100,6 +100,11 @@ public interface ManagedLedgerMXBean {
      */
     long getReadEntriesErrors();
 
+    /**
+     * @return the number of readEntries requests that cache miss Rate
+     */
+    double getReadEntriesOpsCacheMissesRate();
+
     // Entry size statistics
 
     double getEntrySizeAverage();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
index dad101c9b72d1..cb3d72cc5972f 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
@@ -39,6 +39,7 @@ public class ManagedLedgerMBeanImpl implements ManagedLedgerMXBean {
     private final Rate addEntryOpsFailed = new Rate();
     private final Rate readEntriesOps = new Rate();
     private final Rate readEntriesOpsFailed = new Rate();
+    private final Rate readEntriesOpsCacheMisses = new Rate();
     private final Rate markDeleteOps = new Rate();
 
     private final LongAdder dataLedgerOpenOp = new LongAdder();
@@ -72,6 +73,7 @@ public void refreshStats(long period, TimeUnit unit) {
         addEntryOpsFailed.calculateRate(seconds);
         readEntriesOps.calculateRate(seconds);
         readEntriesOpsFailed.calculateRate(seconds);
+        readEntriesOpsCacheMisses.calculateRate(seconds);
         markDeleteOps.calculateRate(seconds);
 
         addEntryLatencyStatsUsec.refresh();
@@ -98,6 +100,10 @@ public void recordReadEntriesError() {
         readEntriesOpsFailed.recordEvent();
     }
 
+    public void recordReadEntriesOpsCacheMisses() {
+        readEntriesOpsCacheMisses.recordEvent();
+    }
+
     public void addAddEntryLatencySample(long latency, TimeUnit unit) {
         addEntryLatencyStatsUsec.addValue(unit.toMicros(latency));
     }
@@ -228,6 +234,11 @@ public long getReadEntriesErrors() {
         return readEntriesOpsFailed.getCount();
     }
 
+    @Override
+    public double getReadEntriesOpsCacheMissesRate() {
+        return readEntriesOpsCacheMisses.getRate();
+    }
+
     @Override
     public double getMarkDeleteRate() {
         return markDeleteOps.getRate();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
index 1c5563b38b120..d2add99b701ac 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
@@ -93,6 +93,7 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole
                     } finally {
                         ledgerEntries.close();
                     }
+                    ml.getMbean().recordReadEntriesOpsCacheMisses();
                     ml.getFactory().getMbean().recordCacheMiss(entries.size(), totalSize);
                     ml.getMbean().addReadEntriesSample(entries.size(), totalSize);
 
@@ -120,6 +121,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks.
                             LedgerEntry ledgerEntry = iterator.next();
                             EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
 
+                            ml.getMbean().recordReadEntriesOpsCacheMisses();
                             ml.getFactory().getMbean().recordCacheMiss(1, returnEntry.getLength());
                             ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
                             callback.readEntryComplete(returnEntry, ctx);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
index 28a2f00cf683c..7747f9bcd93b6 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
@@ -256,6 +256,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt
                                 LedgerEntry ledgerEntry = iterator.next();
                                 EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
 
+                                ml.getMbean().recordReadEntriesOpsCacheMisses();
                                 manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength());
                                 ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
                                 callback.readEntryComplete(returnEntry, ctx);
@@ -449,6 +450,7 @@ CompletableFuture> readFromStorage(ReadHandle lh,
                                     }
                                 }
 
+                                ml.getMbean().recordReadEntriesOpsCacheMisses();
                                 manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize);
                                 ml.getMbean().addReadEntriesSample(entriesToReturn.size(), totalSize);
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java
index 6d82ec682ea1a..36004bc1281bb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java
@@ -108,6 +108,8 @@ private List aggregate(Map> ledgersByD
                         (double) lStats.getReadEntriesErrors());
                 populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesRate",
                         lStats.getReadEntriesRate());
+                populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesOpsCacheMissesRate",
+                        lStats.getReadEntriesOpsCacheMissesRate());
                 populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesSucceeded",
                         (double) lStats.getReadEntriesSucceeded());
                 populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_StoredMessagesSize",
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
index 00c6cecdbfca1..715231d3c6ee1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
@@ -31,6 +31,7 @@ public class AggregatedBrokerStats {
     public long storageLogicalSize;
     public double storageWriteRate;
     public double storageReadRate;
+    public double storageReadCacheMissesRate;
     public long msgBacklog;
 
     void updateStats(TopicStats stats) {
@@ -46,6 +47,7 @@ void updateStats(TopicStats stats) {
         storageLogicalSize += stats.managedLedgerStats.storageLogicalSize;
         storageWriteRate += stats.managedLedgerStats.storageWriteRate;
         storageReadRate += stats.managedLedgerStats.storageReadRate;
+        storageReadCacheMissesRate += stats.managedLedgerStats.storageReadCacheMissesRate;
         msgBacklog += stats.msgBacklog;
     }
 
@@ -62,6 +64,7 @@ public void reset() {
         storageLogicalSize = 0;
         storageWriteRate = 0;
         storageReadRate = 0;
+        storageReadCacheMissesRate = 0;
         msgBacklog = 0;
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
index ea77bd69302a0..9fe5588044d2f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
@@ -107,6 +107,7 @@ void updateStats(TopicStats stats) {
 
         managedLedgerStats.storageWriteRate += stats.managedLedgerStats.storageWriteRate;
         managedLedgerStats.storageReadRate += stats.managedLedgerStats.storageReadRate;
+        managedLedgerStats.storageReadCacheMissesRate += stats.managedLedgerStats.storageReadCacheMissesRate;
 
         msgBacklog += stats.msgBacklog;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java
index db807b51df49f..659f4441adbb7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/ManagedLedgerStats.java
@@ -34,11 +34,13 @@ public class ManagedLedgerStats {
 
     double storageWriteRate;
     double storageReadRate;
+    double storageReadCacheMissesRate;
 
     public void reset() {
         storageSize = 0;
         storageWriteRate = 0;
         storageReadRate = 0;
+        storageReadCacheMissesRate = 0;
         backlogSize = 0;
         offloadedStorageUsed = 0;
         storageLogicalSize = 0;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
index 32fb06ea3ce8c..4e72fa0d72b16 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
@@ -190,6 +190,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include
 
             stats.managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate();
             stats.managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate();
+            stats.managedLedgerStats.storageReadCacheMissesRate = mlStats.getReadEntriesOpsCacheMissesRate();
         }
         TopicStatsImpl tStatus = topic.getStats(getPreciseBacklog, subscriptionBacklogSize, false);
         stats.msgInCounter = tStatus.msgInCounter;
@@ -331,6 +332,8 @@ private static void printBrokerStats(PrometheusMetricStreams stream, String clus
         writeMetric(stream, "pulsar_broker_storage_logical_size", brokerStats.storageLogicalSize, cluster);
         writeMetric(stream, "pulsar_broker_storage_write_rate", brokerStats.storageWriteRate, cluster);
         writeMetric(stream, "pulsar_broker_storage_read_rate", brokerStats.storageReadRate, cluster);
+        writeMetric(stream, "pulsar_broker_storage_read_cache_misses_rate",
+                brokerStats.storageReadCacheMissesRate, cluster);
         writeMetric(stream, "pulsar_broker_msg_backlog", brokerStats.msgBacklog, cluster);
     }
 
@@ -376,6 +379,8 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat
                 cluster, namespace);
         writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate,
                 cluster, namespace);
+        writeMetric(stream, "pulsar_storage_read_cache_misses_rate",
+                stats.managedLedgerStats.storageReadCacheMissesRate, cluster, namespace);
 
         writeMetric(stream, "pulsar_subscription_delayed", stats.msgDelayed, cluster, namespace);
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
index 3a2563a87587b..dda03e3e59dd4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
@@ -152,6 +152,9 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
                 cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
         writeMetric(stream, "pulsar_storage_read_rate", stats.managedLedgerStats.storageReadRate,
                 cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
+        writeMetric(stream, "pulsar_storage_read_cache_misses_rate",
+                stats.managedLedgerStats.storageReadCacheMissesRate,
+                cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
         writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize,
                 cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
         writeMetric(stream, "pulsar_publish_rate_limit_times", stats.publishRateLimitedTimes,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index 13e67762ace6f..04b0a5509b68c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -58,6 +58,7 @@
 import javax.crypto.SecretKey;
 import javax.naming.AuthenticationException;
 import lombok.Cleanup;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -84,6 +85,7 @@
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
@@ -540,6 +542,122 @@ public void testPerTopicStatsReconnect() throws Exception {
         assertEquals(cm.get(0).tags.get("subscription"), "test");
     }
 
+    @DataProvider(name = "cacheEnable")
+    public static Object[][] cacheEnable() {
+        return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } };
+    }
+
+    @Test(dataProvider = "cacheEnable")
+    public void testStorageReadCacheMissesRate(boolean cacheEnable) throws Exception {
+        cleanup();
+        conf.setManagedLedgerStatsPeriodSeconds(Integer.MAX_VALUE);
+        conf.setManagedLedgerCacheEvictionTimeThresholdMillis(Long.MAX_VALUE);
+        conf.setCacheEvictionByMarkDeletedPosition(true);
+        if (cacheEnable) {
+            conf.setManagedLedgerCacheSizeMB(1);
+        } else {
+            conf.setManagedLedgerCacheSizeMB(0);
+        }
+        setup();
+        String ns = "prop/ns-abc1";
+        admin.namespaces().createNamespace(ns);
+        String topic = "persistent://" + ns + "/testStorageReadCacheMissesRate" + UUID.randomUUID();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer().enableBatching(false).topic(topic).create();
+        @Cleanup
+        Consumer consumer = pulsarClient.newConsumer()
+                .topic(topic)
+                .subscriptionName("test")
+                .subscribe();
+        byte[] msg = new byte[2 * 1024 * 1024];
+        new Random().nextBytes(msg);
+        producer.send(msg);
+        consumer.receive();
+        // when cacheEnable, the second msg will read cache miss
+        producer.send(msg);
+        consumer.receive();
+
+        PersistentTopic persistentTopic =
+                (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get();
+        ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) persistentTopic.getManagedLedger());
+        managedLedger.getMbean().refreshStats(1, TimeUnit.SECONDS);
+
+        // includeTopicMetric true
+        ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut);
+        String metricsStr = statsOut.toString();
+        Multimap metrics = parseMetrics(metricsStr);
+
+        metrics.entries().forEach(e -> System.out.println(e.getKey() + ": " + e.getValue()));
+
+        List cm = (List) metrics.get("pulsar_storage_read_cache_misses_rate");
+        assertEquals(cm.size(), 1);
+        if (cacheEnable) {
+            assertEquals(cm.get(0).value, 1.0);
+        } else {
+            assertEquals(cm.get(0).value, 2.0);
+        }
+
+        assertEquals(cm.get(0).tags.get("topic"), topic);
+        assertEquals(cm.get(0).tags.get("namespace"), ns);
+        assertEquals(cm.get(0).tags.get("cluster"), "test");
+
+        List brokerMetric = (List) metrics.get("pulsar_broker_storage_read_cache_misses_rate");
+        assertEquals(brokerMetric.size(), 1);
+        if (cacheEnable) {
+            assertEquals(brokerMetric.get(0).value, 1.0);
+        } else {
+            assertEquals(brokerMetric.get(0).value, 2.0);
+        }
+
+        assertEquals(brokerMetric.get(0).tags.get("cluster"), "test");
+        assertNull(brokerMetric.get(0).tags.get("namespace"));
+        assertNull(brokerMetric.get(0).tags.get("topic"));
+
+        // includeTopicMetric false
+        ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut2);
+        String metricsStr2 = statsOut2.toString();
+        Multimap metrics2 = parseMetrics(metricsStr2);
+
+        metrics2.entries().forEach(e -> System.out.println(e.getKey() + ": " + e.getValue()));
+
+        List cm2 = (List) metrics2.get("pulsar_storage_read_cache_misses_rate");
+        assertEquals(cm2.size(), 1);
+        if (cacheEnable) {
+            assertEquals(cm2.get(0).value, 1.0);
+        } else {
+            assertEquals(cm2.get(0).value, 2.0);
+        }
+
+        assertNull(cm2.get(0).tags.get("topic"));
+        assertEquals(cm2.get(0).tags.get("namespace"), ns);
+        assertEquals(cm2.get(0).tags.get("cluster"), "test");
+
+        List brokerMetric2 = (List) metrics.get("pulsar_broker_storage_read_cache_misses_rate");
+        assertEquals(brokerMetric2.size(), 1);
+        if (cacheEnable) {
+            assertEquals(brokerMetric2.get(0).value, 1.0);
+        } else {
+            assertEquals(brokerMetric2.get(0).value, 2.0);
+        }
+        assertEquals(brokerMetric2.get(0).tags.get("cluster"), "test");
+        assertNull(brokerMetric2.get(0).tags.get("namespace"));
+        assertNull(brokerMetric2.get(0).tags.get("topic"));
+
+        // test ManagedLedgerMetrics
+        List mlMetric = ((List) metrics.get("pulsar_ml_ReadEntriesOpsCacheMissesRate"));
+        assertEquals(mlMetric.size(), 1);
+        if (cacheEnable) {
+            assertEquals(mlMetric.get(0).value, 1.0);
+        } else {
+            assertEquals(mlMetric.get(0).value, 2.0);
+        }
+        assertEquals(mlMetric.get(0).tags.get("cluster"), "test");
+        assertEquals(mlMetric.get(0).tags.get("namespace"), ns + "/persistent");
+    }
+
     @Test
     public void testPerTopicExpiredStat() throws Exception {
         String ns = "prop/ns-abc1";

From 3b118b6bd54cd8e592b8c2cd329a3fa3dfd18a11 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 7 Apr 2023 08:57:19 +0300
Subject: [PATCH 150/404] [fix][sec] Upgrade Spring dependency to 5.3.26 to fix
 OWASP Dependency Check (#20029)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 9f6f2909c1af3..4534036f3fd2c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -232,7 +232,7 @@ flexible messaging model and an intuitive client API.
     1.4.32
     1.0
     9.1.6
-    5.3.20
+    5.3.26
     4.5.13
     4.4.15
     0.5.11

From 55acbe6cef2a0fbd356ad7d085c87e398b7c69db Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Fri, 7 Apr 2023 12:21:12 -0500
Subject: [PATCH 151/404] [refactor][fn] Use AuthorizationServer more in
 Function Worker API (#19975)

### Motivation

The current function worker interfaces introduced by #8560 are hard to extend. It is better to pass an object that we can add fields to as needed.

### Modifications

* Introduce a new `Authentication` class to wrap all of the relevant authentication "data". This class is somewhat unfortunate in that we already have many classes that hold authentication information. However, my goal with this class is to wrap all of the relevant auth info. This class is only currently used in the Function Worker API, but I plan to add it to the rest of the Admin HTTP endpoints.
* Deprecate all of the methods that are no longer needed. Add a default implementation to call the new methods that supersede those methods.
* Add `proxyRoles` setting to function worker.

### Verifying this change

There are tests to cover the changes.

### Does this pull request potentially affect one of the following parts:

This PR changes several interfaces in completely backwards compatible ways. It's possible we'll want to improve the abstraction for the `Authentication` class by making it an interface.

### Documentation

- [x] `doc`

This change has Javadoc updates to document the changes.

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/37
---
 conf/functions_worker.yml                     |   2 +
 .../AuthenticationParameters.java             |  67 +++
 .../authorization/AuthorizationService.java   | 128 +++++-
 .../apache/pulsar/broker/PulsarService.java   |   1 +
 .../broker/admin/impl/BrokerStatsBase.java    |   1 +
 .../broker/admin/impl/FunctionsBase.java      |  49 +--
 .../pulsar/broker/admin/impl/SinksBase.java   |  28 +-
 .../pulsar/broker/admin/impl/SourcesBase.java |  29 +-
 .../pulsar/broker/admin/v2/Functions.java     |  31 +-
 .../apache/pulsar/broker/admin/v2/Worker.java |  20 +-
 .../pulsar/broker/admin/v2/WorkerStats.java   |   4 +-
 .../pulsar/broker/web/PulsarWebResource.java  |   9 +
 .../pulsar/functions/worker/WorkerConfig.java |   6 +
 .../src/test/resources/test_worker_config.yml |   3 +
 .../worker/rest/FunctionApiResource.java      |  18 +
 .../worker/rest/api/ComponentImpl.java        | 411 +++++++-----------
 .../worker/rest/api/FunctionsImpl.java        |  76 ++--
 .../worker/rest/api/FunctionsImplV2.java      |  66 +--
 .../functions/worker/rest/api/SinksImpl.java  |  92 +---
 .../worker/rest/api/SourcesImpl.java          |  92 +---
 .../functions/worker/rest/api/WorkerImpl.java | 105 +++--
 .../rest/api/v2/FunctionsApiV2Resource.java   |  31 +-
 .../rest/api/v2/WorkerApiV2Resource.java      |  36 +-
 .../rest/api/v2/WorkerStatsApiV2Resource.java |  20 +-
 .../rest/api/v3/FunctionsApiV3Resource.java   |  48 +-
 .../rest/api/v3/SinksApiV3Resource.java       |  35 +-
 .../rest/api/v3/SourcesApiV3Resource.java     |  30 +-
 .../worker/service/api/Component.java         | 211 +++------
 .../worker/service/api/Functions.java         |  86 +---
 .../worker/service/api/FunctionsV2.java       |  47 +-
 .../functions/worker/service/api/Sinks.java   |  80 +---
 .../functions/worker/service/api/Sources.java |  82 +---
 .../functions/worker/service/api/Workers.java |  21 +-
 .../functions/worker/WorkerUtilsTest.java     |  14 +
 .../worker/rest/api/FunctionsImplTest.java    | 150 ++++---
 .../worker/rest/api/WorkerImplTest.java       | 120 +++++
 .../api/v2/FunctionApiV2ResourceTest.java     |  40 +-
 .../api/v3/FunctionApiV3ResourceTest.java     |  53 +--
 .../rest/api/v3/SinkApiV3ResourceTest.java    |  43 +-
 .../rest/api/v3/SourceApiV3ResourceTest.java  |  34 +-
 40 files changed, 1169 insertions(+), 1250 deletions(-)
 create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java
 create mode 100644 pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java

diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index b41ac8f37a44f..bb15e0ca416da 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -311,6 +311,8 @@ authenticationProviders:
 authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider
 # Set of role names that are treated as "super-user", meaning they will be able to access any admin-api
 superUserRoles:
+# Set of role names that are treated as "proxy" roles. These are the roles that can supply the originalPrincipal.
+proxyRoles:
 
 #### tls configuration for worker service
 # Enable TLS
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java
new file mode 100644
index 0000000000000..638772345bdf3
--- /dev/null
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationParameters.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication;
+
+import lombok.Builder;
+
+/**
+ * A class to collect all the common fields used for authentication. Because the authentication data source is
+ * not always consistent when using the Pulsar Protocol and the Pulsar Proxy
+ * (see 19332), this class is currently restricted
+ * to use only in authenticating HTTP requests.
+ */
+@Builder
+@lombok.Value
+public class AuthenticationParameters {
+
+    /**
+     * The original principal (or role) of the client.
+     * 

+     *     For HTTP Authentication, there are two possibilities. When the client connects directly to the broker, this
+     *     field is null (assuming the client hasn't supplied the original principal header). When the client connects
+     *     through the proxy, the proxy calculates the original principal based on the client's authentication data and
+     *     supplies it in this field.
+     * 

+     */
+    String originalPrincipal;
+
+    /**
+     * The client role.
+     * 

+     *     For HTTP Authentication, there are three possibilities. When the client connects directly to the broker, this
+     *     is the client's role. When the client connects through the proxy using mTLS, this is the role of the proxy.
+     *     In this case, the {@link #originalPrincipal} is also supplied and is the role of the original client as
+     *     determined by the proxy. When the client connects through the proxy using any other form of authentication,
+     *     this is the role of the original client. In this case, the {@link #originalPrincipal} is also the role of
+     *     the original client.
+     * 

+     */
+    String clientRole;
+
+    /**
+     * The authentication data source used to generate the {@link #clientRole}.
+     * 

+     *     For HTTP Authentication, there are three possibilities. When the client connects directly to the broker, this
+     *     is the client's {@link AuthenticationDataSource}. When the client connects through the proxy using mTLS, this
+     *     is the proxy's {@link AuthenticationDataSource}. When the client connects through the proxy using any other
+     *     form of authentication, this is the original client's {@link AuthenticationDataSource}.
+     * 

+     */
+    AuthenticationDataSource clientAuthenticationDataSource;
+}
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index 6fff04b33b618..6c730f20092dd 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -23,10 +23,12 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import javax.ws.rs.core.Response;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicName;
@@ -49,6 +51,7 @@
 public class AuthorizationService {
     private static final Logger log = LoggerFactory.getLogger(AuthorizationService.class);
 
+    private final PulsarResources resources;
     private final AuthorizationProvider provider;
     private final ServiceConfiguration conf;
 
@@ -61,6 +64,7 @@ public AuthorizationService(ServiceConfiguration conf, PulsarResources pulsarRes
                 provider = (AuthorizationProvider) Class.forName(providerClassname)
                         .getDeclaredConstructor().newInstance();
                 provider.initialize(conf, pulsarResources);
+                this.resources = pulsarResources;
                 log.info("{} has been loaded.", providerClassname);
             } else {
                 throw new PulsarServerException("No authorization providers are present.");
@@ -72,6 +76,23 @@ public AuthorizationService(ServiceConfiguration conf, PulsarResources pulsarRes
         }
     }
 
+    public CompletableFuture isSuperUser(AuthenticationParameters authParams) {
+        if (!isValidOriginalPrincipal(authParams)) {
+            return CompletableFuture.completedFuture(false);
+        }
+        if (isProxyRole(authParams.getClientRole())) {
+            CompletableFuture isRoleAuthorizedFuture = isSuperUser(authParams.getClientRole(),
+                    authParams.getClientAuthenticationDataSource());
+            // The current paradigm is to pass the client auth data when we don't have access to the original auth data.
+            CompletableFuture isOriginalAuthorizedFuture = isSuperUser(authParams.getOriginalPrincipal(),
+                    authParams.getClientAuthenticationDataSource());
+            return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture,
+                    (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized);
+        } else {
+            return isSuperUser(authParams.getClientRole(), authParams.getClientAuthenticationDataSource());
+        }
+    }
+
     public CompletableFuture isSuperUser(String user, AuthenticationDataSource authenticationData) {
         return provider.isSuperUser(user, authenticationData, conf);
     }
@@ -279,17 +300,118 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol
 
     public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role,
                                                             AuthenticationDataSource authenticationData) {
-        return provider.allowFunctionOpsAsync(namespaceName, role, authenticationData);
+        return isSuperUserOrAdmin(namespaceName, role, authenticationData)
+                .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin
+                        ? CompletableFuture.completedFuture(true)
+                        : provider.allowFunctionOpsAsync(namespaceName, role, authenticationData)
+                );
+    }
+
+    public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName,
+                                                            AuthenticationParameters authParams) {
+        if (!isValidOriginalPrincipal(authParams)) {
+            return CompletableFuture.completedFuture(false);
+        }
+        if (isProxyRole(authParams.getClientRole())) {
+            CompletableFuture isRoleAuthorizedFuture = allowFunctionOpsAsync(namespaceName,
+                    authParams.getClientRole(), authParams.getClientAuthenticationDataSource());
+            // The current paradigm is to pass the client auth data when we don't have access to the original auth data.
+            CompletableFuture isOriginalAuthorizedFuture = allowFunctionOpsAsync(
+                    namespaceName, authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource());
+            return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture,
+                    (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized);
+        } else {
+            return allowFunctionOpsAsync(namespaceName, authParams.getClientRole(),
+                    authParams.getClientAuthenticationDataSource());
+        }
     }
 
     public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role,
                                                           AuthenticationDataSource authenticationData) {
-        return provider.allowSourceOpsAsync(namespaceName, role, authenticationData);
+        return isSuperUserOrAdmin(namespaceName, role, authenticationData)
+                .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin
+                        ? CompletableFuture.completedFuture(true)
+                        : provider.allowSourceOpsAsync(namespaceName, role, authenticationData)
+                );
+    }
+
+    public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName,
+                                                          AuthenticationParameters authParams) {
+        if (!isValidOriginalPrincipal(authParams)) {
+            return CompletableFuture.completedFuture(false);
+        }
+        if (isProxyRole(authParams.getClientRole())) {
+            CompletableFuture isRoleAuthorizedFuture = allowSourceOpsAsync(namespaceName,
+                    authParams.getClientRole(), authParams.getClientAuthenticationDataSource());
+            // The current paradigm is to pass the client auth data when we don't have access to the original auth data.
+            CompletableFuture isOriginalAuthorizedFuture = allowSourceOpsAsync(
+                    namespaceName, authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource());
+            return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture,
+                    (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized);
+        } else {
+            return allowSourceOpsAsync(namespaceName, authParams.getClientRole(),
+                    authParams.getClientAuthenticationDataSource());
+        }
     }
 
     public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role,
                                                         AuthenticationDataSource authenticationData) {
-        return provider.allowSinkOpsAsync(namespaceName, role, authenticationData);
+        return isSuperUserOrAdmin(namespaceName, role, authenticationData)
+                .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin
+                        ? CompletableFuture.completedFuture(true)
+                        : provider.allowSinkOpsAsync(namespaceName, role, authenticationData)
+                );
+    }
+
+    public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName,
+                                                        AuthenticationParameters authParams) {
+        if (!isValidOriginalPrincipal(authParams)) {
+            return CompletableFuture.completedFuture(false);
+        }
+        if (isProxyRole(authParams.getClientRole())) {
+            CompletableFuture isRoleAuthorizedFuture = allowSinkOpsAsync(namespaceName,
+                    authParams.getClientRole(), authParams.getClientAuthenticationDataSource());
+            // The current paradigm is to pass the client auth data when we don't have access to the original auth data.
+            CompletableFuture isOriginalAuthorizedFuture = allowSinkOpsAsync(
+                    namespaceName, authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource());
+            return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture,
+                    (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized);
+        } else {
+            return allowSinkOpsAsync(namespaceName, authParams.getClientRole(),
+                    authParams.getClientAuthenticationDataSource());
+        }
+    }
+
+    /**
+     * Functions, sources, and sinks each have their own method in this class. This method first checks for
+     * tenant admin access, then for namespace level permission.
+     */
+    private CompletableFuture isSuperUserOrAdmin(NamespaceName namespaceName,
+                                                          String role,
+                                                          AuthenticationDataSource authenticationData) {
+        return isSuperUser(role, authenticationData)
+                .thenCompose(isSuperUserOrAdmin -> isSuperUserOrAdmin
+                        ? CompletableFuture.completedFuture(true)
+                        : isTenantAdmin(namespaceName.getTenant(), role, authenticationData));
+    }
+
+    private CompletableFuture isTenantAdmin(String tenant, String role,
+                                                    AuthenticationDataSource authData) {
+        return resources.getTenantResources()
+                .getTenantAsync(tenant)
+                .thenCompose(op -> {
+                    if (op.isPresent()) {
+                        return isTenantAdmin(tenant, role, op.get(), authData);
+                    } else {
+                        return CompletableFuture.failedFuture(new RestException(Response.Status.NOT_FOUND,
+                                "Tenant does not exist"));
+                    }
+                });
+    }
+
+    private boolean isValidOriginalPrincipal(AuthenticationParameters authParams) {
+        return isValidOriginalPrincipal(authParams.getClientRole(),
+                authParams.getOriginalPrincipal(), authParams.getClientAuthenticationDataSource());
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 4af94c339c82f..b2eefe298b53c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -1860,6 +1860,7 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu
 
         // inherit super users
         workerConfig.setSuperUserRoles(brokerConfig.getSuperUserRoles());
+        workerConfig.setProxyRoles(brokerConfig.getProxyRoles());
         workerConfig.setFunctionsWorkerEnablePackageManagement(brokerConfig.isFunctionsWorkerEnablePackageManagement());
 
         // inherit the nar package locations
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java
index 670fa10b28b88..6d49dd81da13d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java
@@ -159,6 +159,7 @@ public LoadManagerReport getLoadReport() throws Exception {
 
     protected Map> internalBrokerResourceAvailability(NamespaceName namespace) {
         try {
+            validateSuperUserAccess();
             LoadManager lm = pulsar().getLoadManager().get();
             if (lm instanceof SimpleLoadManagerImpl) {
                 return ((SimpleLoadManagerImpl) lm).getResourceAvailabilityFor(namespace).asMap();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
index bee81bf5dcdb6..4350316e2f011 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
@@ -194,7 +194,7 @@ public void registerFunction(
             )
             final @FormDataParam("functionConfig") FunctionConfig functionConfig) {
         functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-            functionPkgUrl, functionConfig, clientAppId(), clientAuthData());
+            functionPkgUrl, functionConfig, authParams());
     }
 
     @PUT
@@ -322,7 +322,7 @@ public void updateFunction(
             final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) throws IOException {
 
         functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionConfig, clientAppId(), clientAuthData(), updateOptions);
+                functionPkgUrl, functionConfig, authParams(), updateOptions);
     }
 
 
@@ -343,7 +343,7 @@ public void deregisterFunction(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) {
-        functions().deregisterFunction(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().deregisterFunction(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -365,7 +365,7 @@ public FunctionConfig getFunctionInfo(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) throws IOException {
-        return functions().getFunctionInfo(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        return functions().getFunctionInfo(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -389,7 +389,7 @@ public FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData getFunct
                     + " the stats of all instances is returned") final @PathParam("instanceId")
                     String instanceId) throws IOException {
         return functions().getFunctionInstanceStatus(tenant, namespace, functionName,
-                instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+                instanceId, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -413,7 +413,7 @@ public FunctionStatus getFunctionStatus(
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) throws IOException {
         return functions().getFunctionStatus(tenant, namespace, functionName, uri.getRequestUri(),
-                clientAppId(), clientAuthData());
+                authParams());
     }
 
     @GET
@@ -437,7 +437,7 @@ public FunctionStatsImpl getFunctionStats(
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) throws IOException {
         return functions().getFunctionStats(tenant, namespace, functionName,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -461,7 +461,7 @@ public FunctionInstanceStatsDataImpl getFunctionInstanceStats(
                     + " (if instance-id is not provided, the stats of all instances is returned") final @PathParam(
                     "instanceId") String instanceId) throws IOException {
         return functions().getFunctionsInstanceStats(tenant, namespace, functionName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -480,7 +480,7 @@ public List listFunctions(
             final @PathParam("tenant") String tenant,
             @ApiParam(value = "The namespace of a Pulsar Function")
             final @PathParam("namespace") String namespace) {
-        return functions().listFunctions(tenant, namespace, clientAppId(), clientAuthData());
+        return functions().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -509,7 +509,7 @@ public String triggerFunction(
                     + " consumes from which you want to inject the data to") final @FormDataParam("topic")
                     String topic) {
         return functions().triggerFunction(tenant, namespace, functionName, triggerValue,
-                triggerStream, topic, clientAppId(), clientAuthData());
+                triggerStream, topic, authParams());
     }
 
     @GET
@@ -533,7 +533,7 @@ public FunctionState getFunctionState(
             final @PathParam("functionName") String functionName,
             @ApiParam(value = "The stats key")
             final @PathParam("key") String key) {
-        return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId(), clientAuthData());
+        return functions().getFunctionState(tenant, namespace, functionName, key, authParams());
     }
 
     @POST
@@ -553,7 +553,7 @@ public void putFunctionState(final @PathParam("tenant") String tenant,
                                  final @PathParam("functionName") String functionName,
                                  final @PathParam("key") String key,
                                  final @FormDataParam("state") FunctionState stateJson) {
-        functions().putFunctionState(tenant, namespace, functionName, key, stateJson, clientAppId(), clientAuthData());
+        functions().putFunctionState(tenant, namespace, functionName, key, stateJson, authParams());
     }
 
     @POST
@@ -574,7 +574,7 @@ public void restartFunction(
                     "The instanceId of a Pulsar Function (if instance-id is not provided, all instances are restarted")
             final @PathParam("instanceId") String instanceId) {
         functions().restartFunctionInstance(tenant, namespace, functionName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -593,7 +593,7 @@ public void restartFunction(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) {
-        functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().restartFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -613,7 +613,7 @@ public void stopFunction(
                     "The instanceId of a Pulsar Function (if instance-id is not provided, all instances are stopped. ")
             final @PathParam("instanceId") String instanceId) {
         functions().stopFunctionInstance(tenant, namespace, functionName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -632,7 +632,7 @@ public void stopFunction(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) {
-        functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().stopFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -652,7 +652,7 @@ public void startFunction(
                     + " (if instance-id is not provided, all instances sre started. ") final @PathParam("instanceId")
                     String instanceId) {
         functions().startFunctionInstance(tenant, namespace, functionName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -671,7 +671,7 @@ public void startFunction(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Function")
             final @PathParam("functionName") String functionName) {
-        functions().startFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().startFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -683,7 +683,7 @@ public void startFunction(
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     public void uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
                                final @FormDataParam("path") String path) {
-        functions().uploadFunction(uploadedInputStream, path, clientAppId(), clientAuthData());
+        functions().uploadFunction(uploadedInputStream, path, authParams());
     }
 
     @GET
@@ -693,7 +693,7 @@ public void uploadFunction(final @FormDataParam("data") InputStream uploadedInpu
     )
     @Path("/download")
     public StreamingOutput downloadFunction(final @QueryParam("path") String path) {
-        return functions().downloadFunction(path, clientAppId(), clientAuthData());
+        return functions().downloadFunction(path, authParams());
     }
 
     @GET
@@ -712,8 +712,7 @@ public StreamingOutput downloadFunction(
             @ApiParam(value = "Whether to download the transform-function")
             final @QueryParam("transform-function") boolean transformFunction) {
 
-        return functions()
-                .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), transformFunction);
+        return functions().downloadFunction(tenant, namespace, functionName, authParams(), transformFunction);
     }
 
     @GET
@@ -746,7 +745,7 @@ public List getConnectorsList() throws IOException {
     })
     @Path("/builtins/reload")
     public void reloadBuiltinFunctions() throws IOException {
-        functions().reloadBuiltinFunctions(clientAppId(), clientAuthData());
+        functions().reloadBuiltinFunctions(authParams());
     }
 
     @GET
@@ -763,7 +762,7 @@ public void reloadBuiltinFunctions() throws IOException {
     @Path("/builtins")
     @Produces(MediaType.APPLICATION_JSON)
     public List getBuiltinFunction() {
-        return functions().getBuiltinFunctions(clientAppId(), clientAuthData());
+        return functions().getBuiltinFunctions(authParams());
     }
 
     @PUT
@@ -785,6 +784,6 @@ public void updateFunctionOnWorkerLeader(final @PathParam("tenant") String tenan
                                              final @FormDataParam("delete") boolean delete) {
 
         functions().updateFunctionOnWorkerLeader(tenant, namespace, functionName, uploadedInputStream,
-                delete, uri.getRequestUri(), clientAppId(), clientAuthData());
+                delete, uri.getRequestUri(), authParams());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java
index 7a3a554a3a00c..80ad72d6f9aa9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java
@@ -168,7 +168,7 @@ public void registerSink(@ApiParam(value = "The tenant of a Pulsar Sink") final
                              )
                              final @FormDataParam("sinkConfig") SinkConfig sinkConfig) {
         sinks().registerSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail,
-                sinkPkgUrl, sinkConfig, clientAppId(), clientAuthData());
+                sinkPkgUrl, sinkConfig, authParams());
     }
 
     @PUT
@@ -271,7 +271,7 @@ public void updateSink(@ApiParam(value = "The tenant of a Pulsar Sink") final @P
                            @ApiParam(value = "Update options for the Pulsar Sink")
                            final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) {
          sinks().updateSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail,
-                sinkPkgUrl, sinkConfig, clientAppId(), clientAuthData(), updateOptions);
+                sinkPkgUrl, sinkConfig, authParams(), updateOptions);
 
     }
 
@@ -295,7 +295,7 @@ public void deregisterSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                                final @PathParam("namespace") String namespace,
                                @ApiParam(value = "The name of a Pulsar Sink")
                                final @PathParam("sinkName") String sinkName) {
-        sinks().deregisterFunction(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().deregisterFunction(tenant, namespace, sinkName, authParams());
     }
 
     @GET
@@ -315,7 +315,7 @@ public SinkConfig getSinkInfo(@ApiParam(value = "The tenant of a Pulsar Sink")
                                   final @PathParam("namespace") String namespace,
                                   @ApiParam(value = "The name of a Pulsar Sink")
                                   final @PathParam("sinkName") String sinkName) throws IOException {
-        return sinks().getSinkInfo(tenant, namespace, sinkName);
+        return sinks().getSinkInfo(tenant, namespace, sinkName, authParams());
     }
 
     @GET
@@ -342,7 +342,7 @@ public SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkInstanceStatu
             @ApiParam(value = "The instanceId of a Pulsar Sink")
             final @PathParam("instanceId") String instanceId) throws IOException {
         return sinks().getSinkInstanceStatus(
-            tenant, namespace, sinkName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+            tenant, namespace, sinkName, instanceId, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -365,7 +365,7 @@ public SinkStatus getSinkStatus(@ApiParam(value = "The tenant of a Pulsar Sink")
                                     final @PathParam("namespace") String namespace,
                                     @ApiParam(value = "The name of a Pulsar Sink")
                                     final @PathParam("sinkName") String sinkName) throws IOException {
-        return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), clientAppId(), clientAuthData());
+        return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -385,7 +385,7 @@ public List listSinks(@ApiParam(value = "The tenant of a Pulsar Sink")
                                   final @PathParam("tenant") String tenant,
                                   @ApiParam(value = "The namespace of a Pulsar Sink")
                                   final @PathParam("namespace") String namespace) {
-        return sinks().listFunctions(tenant, namespace, clientAppId(), clientAuthData());
+        return sinks().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -411,7 +411,7 @@ public void restartSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                             @ApiParam(value = "The instanceId of a Pulsar Sink")
                             final @PathParam("instanceId") String instanceId) {
         sinks().restartFunctionInstance(tenant, namespace, sinkName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -432,7 +432,7 @@ public void restartSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                             final @PathParam("namespace") String namespace,
                             @ApiParam(value = "The name of a Pulsar Sink")
                             final @PathParam("sinkName") String sinkName) {
-        sinks().restartFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().restartFunctionInstances(tenant, namespace, sinkName, authParams());
     }
 
     @POST
@@ -456,7 +456,7 @@ public void stopSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                          @ApiParam(value = "The instanceId of a Pulsar Sink")
                          final @PathParam("instanceId") String instanceId) {
         sinks().stopFunctionInstance(tenant, namespace,
-                sinkName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+                sinkName, instanceId, uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -477,7 +477,7 @@ public void stopSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                          final @PathParam("namespace") String namespace,
                          @ApiParam(value = "The name of a Pulsar Sink")
                          final @PathParam("sinkName") String sinkName) {
-        sinks().stopFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().stopFunctionInstances(tenant, namespace, sinkName, authParams());
     }
 
     @POST
@@ -501,7 +501,7 @@ public void startSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                           @ApiParam(value = "The instanceId of a Pulsar Sink")
                           final @PathParam("instanceId") String instanceId) {
         sinks().startFunctionInstance(tenant, namespace, sinkName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -522,7 +522,7 @@ public void startSink(@ApiParam(value = "The tenant of a Pulsar Sink")
                           final @PathParam("namespace") String namespace,
                           @ApiParam(value = "The name of a Pulsar Sink")
                           final @PathParam("sinkName") String sinkName) {
-        sinks().startFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().startFunctionInstances(tenant, namespace, sinkName, authParams());
     }
 
     @GET
@@ -571,6 +571,6 @@ public List getSinkConfigDefinition(
     })
     @Path("/reloadBuiltInSinks")
     public void reloadSinks() {
-        sinks().reloadConnectors(clientAppId(), clientAuthData());
+        sinks().reloadConnectors(authParams());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java
index 1bf092784dd3a..4af0afc0d6ec5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java
@@ -142,7 +142,7 @@ public void registerSource(
             )
             final @FormDataParam("sourceConfig") SourceConfig sourceConfig) {
         sources().registerSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail,
-            sourcePkgUrl, sourceConfig, clientAppId(), clientAuthData());
+            sourcePkgUrl, sourceConfig, authParams());
     }
 
     @PUT
@@ -227,7 +227,7 @@ public void updateSource(
             @ApiParam(value = "Update options for Pulsar Source")
             final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) {
         sources().updateSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail,
-            sourcePkgUrl, sourceConfig, clientAppId(), clientAuthData(), updateOptions);
+            sourcePkgUrl, sourceConfig, authParams(), updateOptions);
     }
 
 
@@ -250,7 +250,7 @@ public void deregisterSource(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Source")
             final @PathParam("sourceName") String sourceName) {
-        sources().deregisterFunction(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().deregisterFunction(tenant, namespace, sourceName, authParams());
     }
 
     @GET
@@ -271,7 +271,7 @@ public SourceConfig getSourceInfo(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Source")
             final @PathParam("sourceName") String sourceName) throws IOException {
-        return sources().getSourceInfo(tenant, namespace, sourceName);
+        return sources().getSourceInfo(tenant, namespace, sourceName, authParams());
     }
 
     @GET
@@ -294,7 +294,7 @@ public SourceStatus.SourceInstanceStatus.SourceInstanceStatusData getSourceInsta
                     + " (if instance-id is not provided, the stats of all instances is returned).") final @PathParam(
                     "instanceId") String instanceId) throws IOException {
         return sources().getSourceInstanceStatus(
-            tenant, namespace, sourceName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+            tenant, namespace, sourceName, instanceId, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -316,8 +316,7 @@ public SourceStatus getSourceStatus(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Source")
             final @PathParam("sourceName") String sourceName) throws IOException {
-        return sources().getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), clientAppId(),
-                clientAuthData());
+        return sources().getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -339,7 +338,7 @@ public List listSources(
             final @PathParam("tenant") String tenant,
             @ApiParam(value = "The namespace of a Pulsar Source")
             final @PathParam("namespace") String namespace) {
-        return sources().listFunctions(tenant, namespace, clientAppId(), clientAuthData());
+        return sources().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -362,7 +361,7 @@ public void restartSource(
                     + " (if instance-id is not provided, the stats of all instances is returned).") final @PathParam(
                     "instanceId") String instanceId) {
         sources().restartFunctionInstance(tenant, namespace, sourceName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -383,7 +382,7 @@ public void restartSource(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Source")
             final @PathParam("sourceName") String sourceName) {
-        sources().restartFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().restartFunctionInstances(tenant, namespace, sourceName, authParams());
     }
 
     @POST
@@ -404,7 +403,7 @@ public void stopSource(
             @ApiParam(value = "The instanceId of a Pulsar Source (if instance-id is not provided,"
                     + " the stats of all instances is returned).") final @PathParam("instanceId") String instanceId) {
         sources().stopFunctionInstance(tenant, namespace, sourceName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -425,7 +424,7 @@ public void stopSource(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Source")
             final @PathParam("sourceName") String sourceName) {
-        sources().stopFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().stopFunctionInstances(tenant, namespace, sourceName, authParams());
     }
 
     @POST
@@ -446,7 +445,7 @@ public void startSource(
             @ApiParam(value = "The instanceId of a Pulsar Source (if instance-id is not provided,"
                     + " the stats of all instances is returned).") final @PathParam("instanceId") String instanceId) {
         sources().startFunctionInstance(tenant, namespace, sourceName, instanceId,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -467,7 +466,7 @@ public void startSource(
             final @PathParam("namespace") String namespace,
             @ApiParam(value = "The name of a Pulsar Source")
             final @PathParam("sourceName") String sourceName) {
-        sources().startFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().startFunctionInstances(tenant, namespace, sourceName, authParams());
     }
 
     @GET
@@ -520,6 +519,6 @@ public List getSourceConfigDefinition(
     })
     @Path("/reloadBuiltInSources")
     public void reloadSources() {
-        sources().reloadConnectors(clientAppId(), clientAuthData());
+        sources().reloadConnectors(authParams());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java
index 0d302bec065bc..69f99a42d0685 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java
@@ -75,7 +75,7 @@ public Response registerFunction(final @PathParam("tenant") String tenant,
                                      final @FormDataParam("functionDetails") String functionDetailsJson) {
 
         return functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionDetailsJson, clientAppId());
+                functionPkgUrl, functionDetailsJson, authParams());
     }
 
     @PUT
@@ -96,7 +96,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant,
                                    final @FormDataParam("functionDetails") String functionDetailsJson) {
 
         return functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionDetailsJson, clientAppId());
+                functionPkgUrl, functionDetailsJson, authParams());
     }
 
 
@@ -113,7 +113,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant,
     public Response deregisterFunction(final @PathParam("tenant") String tenant,
                                        final @PathParam("namespace") String namespace,
                                        final @PathParam("functionName") String functionName) {
-        return functions().deregisterFunction(tenant, namespace, functionName, clientAppId());
+        return functions().deregisterFunction(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -132,8 +132,7 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace,
                                     final @PathParam("functionName") String functionName) throws IOException {
 
-        return functions().getFunctionInfo(
-                tenant, namespace, functionName, clientAppId());
+        return functions().getFunctionInfo(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -154,7 +153,7 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena
                                               final @PathParam("instanceId") String instanceId) throws IOException {
 
         return functions().getFunctionInstanceStatus(tenant, namespace, functionName, instanceId, uri.getRequestUri(),
-                clientAppId());
+                authParams());
     }
 
     @GET
@@ -172,7 +171,7 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant,
                                       final @PathParam("namespace") String namespace,
                                       final @PathParam("functionName") String functionName) throws IOException {
         return functions().getFunctionStatusV2(
-                tenant, namespace, functionName, uri.getRequestUri(), clientAppId());
+                tenant, namespace, functionName, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -188,7 +187,7 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant,
     @Path("/{tenant}/{namespace}")
     public Response listFunctions(final @PathParam("tenant") String tenant,
                                   final @PathParam("namespace") String namespace) {
-        return functions().listFunctions(tenant, namespace, clientAppId());
+        return functions().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -211,7 +210,7 @@ public Response triggerFunction(final @PathParam("tenant") String tenant,
                                     final @FormDataParam("dataStream") InputStream triggerStream,
                                     final @FormDataParam("topic") String topic) {
         return functions().triggerFunction(tenant, namespace, functionName,
-                triggerValue, triggerStream, topic, clientAppId());
+                triggerValue, triggerStream, topic, authParams());
     }
 
     @GET
@@ -230,7 +229,7 @@ public Response getFunctionState(final @PathParam("tenant") String tenant,
                                      final @PathParam("namespace") String namespace,
                                      final @PathParam("functionName") String functionName,
                                      final @PathParam("key") String key) {
-        return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId());
+        return functions().getFunctionState(tenant, namespace, functionName, key, authParams());
     }
 
     @POST
@@ -247,7 +246,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant,
                                     final @PathParam("functionName") String functionName,
                                     final @PathParam("instanceId") String instanceId) {
         return functions().restartFunctionInstance(tenant, namespace, functionName,
-                instanceId, uri.getRequestUri(), clientAppId());
+                instanceId, uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -260,7 +259,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant,
     public Response restartFunction(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace,
                                     final @PathParam("functionName") String functionName) {
-        return functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId());
+        return functions().restartFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -275,7 +274,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant,
                                  final @PathParam("functionName") String functionName,
                                  final @PathParam("instanceId") String instanceId) {
         return functions().stopFunctionInstance(tenant, namespace, functionName,
-                instanceId, uri.getRequestUri(), clientAppId());
+                instanceId, uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -288,7 +287,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant,
     public Response stopFunction(final @PathParam("tenant") String tenant,
                                  final @PathParam("namespace") String namespace,
                                  final @PathParam("functionName") String functionName) {
-        return functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId());
+        return functions().stopFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -300,7 +299,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant,
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
                                    final @FormDataParam("path") String path) {
-        return functions().uploadFunction(uploadedInputStream, path, clientAppId());
+        return functions().uploadFunction(uploadedInputStream, path, authParams());
     }
 
     @GET
@@ -310,7 +309,7 @@ public Response uploadFunction(final @FormDataParam("data") InputStream uploaded
     )
     @Path("/download")
     public Response downloadFunction(final @QueryParam("path") String path) {
-        return functions().downloadFunction(path, clientAppId());
+        return functions().downloadFunction(path, authParams());
     }
 
     @GET
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java
index 80c75b04917ce..3813790e4f428 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java
@@ -66,7 +66,7 @@ public WorkerService get() {
     @Path("/cluster")
     @Produces(MediaType.APPLICATION_JSON)
     public List getCluster() {
-        return workers().getCluster(clientAppId());
+        return workers().getCluster(authParams());
     }
 
     @GET
@@ -81,7 +81,7 @@ public List getCluster() {
     @Path("/cluster/leader")
     @Produces(MediaType.APPLICATION_JSON)
     public WorkerInfo getClusterLeader() {
-        return workers().getClusterLeader(clientAppId());
+        return workers().getClusterLeader(authParams());
     }
 
     @GET
@@ -96,7 +96,7 @@ public WorkerInfo getClusterLeader() {
     @Path("/assignments")
     @Produces(MediaType.APPLICATION_JSON)
     public Map> getAssignments() {
-        return workers().getAssignments(clientAppId());
+        return workers().getAssignments(authParams());
     }
 
     @GET
@@ -112,7 +112,7 @@ public Map> getAssignments() {
     @Path("/connectors")
     @Produces(MediaType.APPLICATION_JSON)
     public List getConnectorsList() throws IOException {
-        return workers().getListOfConnectors(clientAppId());
+        return workers().getListOfConnectors(authParams());
     }
 
     @PUT
@@ -126,7 +126,7 @@ public List getConnectorsList() throws IOException {
     })
     @Path("/rebalance")
     public void rebalance() {
-        workers().rebalance(uri.getRequestUri(), clientAppId());
+        workers().rebalance(uri.getRequestUri(), authParams());
     }
 
     @PUT
@@ -142,7 +142,7 @@ public void rebalance() {
     })
     @Path("/leader/drain")
     public void drainAtLeader(@QueryParam("workerId") String workerId) {
-        workers().drain(uri.getRequestUri(), workerId, clientAppId(), true);
+        workers().drain(uri.getRequestUri(), workerId, authParams(), true);
     }
 
     @PUT
@@ -158,7 +158,7 @@ public void drainAtLeader(@QueryParam("workerId") String workerId) {
     })
     @Path("/drain")
     public void drain() {
-        workers().drain(uri.getRequestUri(), null, clientAppId(), false);
+        workers().drain(uri.getRequestUri(), null, authParams(), false);
     }
 
     @GET
@@ -172,7 +172,7 @@ public void drain() {
     })
     @Path("/leader/drain")
     public LongRunningProcessStatus getDrainStatusFromLeader(@QueryParam("workerId") String workerId) {
-        return workers().getDrainStatus(uri.getRequestUri(), workerId, clientAppId(), true);
+        return workers().getDrainStatus(uri.getRequestUri(), workerId, authParams(), true);
     }
 
     @GET
@@ -186,7 +186,7 @@ public LongRunningProcessStatus getDrainStatusFromLeader(@QueryParam("workerId")
     })
     @Path("/drain")
     public LongRunningProcessStatus getDrainStatus() {
-        return workers().getDrainStatus(uri.getRequestUri(), null, clientAppId(), false);
+        return workers().getDrainStatus(uri.getRequestUri(), null, authParams(), false);
     }
 
     @GET
@@ -199,6 +199,6 @@ public LongRunningProcessStatus getDrainStatus() {
     })
     @Path("/cluster/leader/ready")
     public Boolean isLeaderReady() {
-        return workers().isLeaderReady(clientAppId());
+        return workers().isLeaderReady(authParams());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java
index 5a77d81b6f1e3..1716b5e588c77 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/WorkerStats.java
@@ -56,7 +56,7 @@ public Workers workers() {
     })
     @Produces(MediaType.APPLICATION_JSON)
     public Collection getMetrics() throws Exception {
-        return workers().getWorkerMetrics(clientAppId());
+        return workers().getWorkerMetrics(authParams());
     }
 
     @GET
@@ -72,6 +72,6 @@ public Collection getMetrics() throws Exception {
     })
     @Produces(MediaType.APPLICATION_JSON)
     public List getStats() throws IOException {
-        return workers().getFunctionsMetrics(clientAppId());
+        return workers().getFunctionsMetrics(authParams());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
index a182e4733fdfe..3d23d7812543a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.authorization.AuthorizationService;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.namespace.LookupOptions;
@@ -143,6 +144,14 @@ public static String splitPath(String source, int slice) {
         return PolicyPath.splitPath(source, slice);
     }
 
+    public AuthenticationParameters authParams() {
+        return AuthenticationParameters.builder()
+                .originalPrincipal(originalPrincipal())
+                .clientRole(clientAppId())
+                .clientAuthenticationDataSource(clientAuthData())
+                .build();
+    }
+
     /**
      * Gets a caller id (IP + role).
      *
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
index b87477f78c641..3b8ddf774d162 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
@@ -563,6 +563,12 @@ public boolean isBrokerClientAuthenticationEnabled() {
     )
     private Set superUserRoles = new TreeSet<>();
 
+    @FieldContext(
+            category = CATEGORY_WORKER_SECURITY,
+            doc = "Role names that are treated as `proxy roles`. These are the only roles that can supply the "
+                    + "originalPrincipal.")
+    private Set proxyRoles = new TreeSet<>();
+
     @FieldContext(
             category = CATEGORY_WORKER_SECURITY,
             doc = "This is a regexp, which limits the range of possible ids which can connect to the Broker using SASL."
diff --git a/pulsar-functions/src/test/resources/test_worker_config.yml b/pulsar-functions/src/test/resources/test_worker_config.yml
index f0ecf2bd71bc6..a297788037035 100644
--- a/pulsar-functions/src/test/resources/test_worker_config.yml
+++ b/pulsar-functions/src/test/resources/test_worker_config.yml
@@ -23,6 +23,9 @@ pulsarServiceUrl: pulsar://localhost:6650
 functionMetadataTopicName: test-function-metadata-topic
 numFunctionPackageReplicas: 3
 maxPendingAsyncRequests: 200
+proxyRoles:
+  - "proxyA"
+  - "proxyB"
 properties:
   # Fake Bookkeeper Client config to be applied to the DLog Bookkeeper Client
   bookkeeper_testKey: "fakeValue"
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java
index d37f7f15c606e..cfce3d3e68912 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java
@@ -24,12 +24,14 @@
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.UriInfo;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.web.AuthenticationFilter;
 import org.apache.pulsar.functions.worker.WorkerService;
 
 public class FunctionApiResource implements Supplier {
 
     public static final String ATTRIBUTE_FUNCTION_WORKER = "function-worker";
+    public static final String ORIGINAL_PRINCIPAL_HEADER = "X-Original-Principal";
 
     private WorkerService workerService;
     @Context
@@ -47,12 +49,28 @@ public synchronized WorkerService get() {
         return this.workerService;
     }
 
+    public AuthenticationParameters authParams() {
+        return AuthenticationParameters.builder()
+                .originalPrincipal(httpRequest.getHeader(ORIGINAL_PRINCIPAL_HEADER))
+                .clientRole(clientAppId())
+                .clientAuthenticationDataSource(clientAuthData())
+                .build();
+    }
+
+    /**
+     * @deprecated use {@link #authParams()} instead.
+     */
+    @Deprecated
     public String clientAppId() {
         return httpRequest != null
                 ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName)
                 : null;
     }
 
+    /**
+     * @deprecated use {@link #authParams()} instead.
+     */
+    @Deprecated
     public AuthenticationDataSource clientAuthData() {
         return (AuthenticationDataSource) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName);
     }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java
index 0b6c2bbd61f55..7daa9c9aee8ed 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java
@@ -67,6 +67,7 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.admin.internal.FunctionsImpl;
 import org.apache.pulsar.client.api.Message;
@@ -84,7 +85,6 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl;
 import org.apache.pulsar.common.policies.data.FunctionStatsImpl;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.apache.pulsar.common.util.Codec;
 import org.apache.pulsar.common.util.RestException;
 import org.apache.pulsar.functions.instance.InstanceUtils;
@@ -456,23 +456,14 @@ private void deleteStatestoreTableAsync(String namespace, String table) {
     public void deregisterFunction(final String tenant,
                                    final String namespace,
                                    final String componentName,
-                                   final String clientRole,
-                                   AuthenticationDataSource clientAuthenticationDataHttps) {
+                                   final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to deregister {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "deregister",
+                authParams);
 
         // validate parameters
         try {
@@ -540,23 +531,14 @@ private void deleteComponentFromStorage(String tenant, String namespace, String
     public FunctionConfig getFunctionInfo(final String tenant,
                                           final String namespace,
                                           final String componentName,
-                                          final String clientRole,
-                                          final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                          final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to get {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get",
+                authParams);
 
         // validate parameters
         try {
@@ -591,10 +573,8 @@ public void stopFunctionInstance(final String tenant,
                                      final String componentName,
                                      final String instanceId,
                                      final URI uri,
-                                     final String clientRole,
-                                     final AuthenticationDataSource clientAuthenticationDataHttps) {
-        changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, false, uri, clientRole,
-                clientAuthenticationDataHttps);
+                                     final AuthenticationParameters authParams) {
+        changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, false, uri, authParams);
     }
 
     @Override
@@ -603,12 +583,11 @@ public void startFunctionInstance(final String tenant,
                                       final String componentName,
                                       final String instanceId,
                                       final URI uri,
-                                      final String clientRole,
-                                      final AuthenticationDataSource clientAuthenticationDataHttps) {
-        changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, true, uri, clientRole,
-                clientAuthenticationDataHttps);
+                                      final AuthenticationParameters authParams) {
+        changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, true, uri, authParams);
     }
 
+    @Deprecated
     public void changeFunctionInstanceStatus(final String tenant,
                                              final String namespace,
                                              final String componentName,
@@ -617,21 +596,27 @@ public void changeFunctionInstanceStatus(final String tenant,
                                              final URI uri,
                                              final String clientRole,
                                              final AuthenticationDataSource clientAuthenticationDataHttps) {
+        AuthenticationParameters authParams = AuthenticationParameters.builder()
+                .clientRole(clientRole)
+                .clientAuthenticationDataSource(clientAuthenticationDataHttps)
+                .build();
+        changeFunctionInstanceStatus(tenant, namespace, componentName, instanceId, start, uri, authParams);
+    }
+
+    public void changeFunctionInstanceStatus(final String tenant,
+                                             final String namespace,
+                                             final String componentName,
+                                             final String instanceId,
+                                             final boolean start,
+                                             final URI uri,
+                                             final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to start/stop {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "start/stop",
+                authParams);
 
         // validate parameters
         try {
@@ -678,22 +663,13 @@ public void restartFunctionInstance(final String tenant,
                                         final String componentName,
                                         final String instanceId,
                                         final URI uri,
-                                        final String clientRole,
-                                        final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                        final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to restart {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "restart",
+                authParams);
 
         // validate parameters
         try {
@@ -738,43 +714,41 @@ public void restartFunctionInstance(final String tenant,
     public void stopFunctionInstances(final String tenant,
                                       final String namespace,
                                       final String componentName,
-                                      final String clientRole,
-                                      final AuthenticationDataSource clientAuthenticationDataHttps) {
-        changeFunctionStatusAllInstances(tenant, namespace, componentName, false, clientRole,
-                clientAuthenticationDataHttps);
+                                      final AuthenticationParameters authParams) {
+        changeFunctionStatusAllInstances(tenant, namespace, componentName, false, authParams);
     }
 
     @Override
     public void startFunctionInstances(final String tenant,
                                        final String namespace,
                                        final String componentName,
-                                       final String clientRole,
-                                       final AuthenticationDataSource clientAuthenticationDataHttps) {
-        changeFunctionStatusAllInstances(tenant, namespace, componentName, true, clientRole,
-                clientAuthenticationDataHttps);
+                                       final AuthenticationParameters authParams) {
+        changeFunctionStatusAllInstances(tenant, namespace, componentName, true, authParams);
     }
 
+    @Deprecated
     public void changeFunctionStatusAllInstances(final String tenant,
                                                  final String namespace,
                                                  final String componentName,
                                                  final boolean start,
                                                  final String clientRole,
                                                  final AuthenticationDataSource clientAuthenticationDataHttps) {
+        AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole)
+                .clientAuthenticationDataSource(clientAuthenticationDataHttps).build();
+        changeFunctionStatusAllInstances(tenant, namespace, componentName, start, authParams);
+    }
 
+    public void changeFunctionStatusAllInstances(final String tenant,
+                                                 final String namespace,
+                                                 final String componentName,
+                                                 final boolean start,
+                                                 final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to start/stop {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "start/stop",
+                authParams);
 
         // validate parameters
         try {
@@ -815,25 +789,17 @@ public void changeFunctionStatusAllInstances(final String tenant,
                         namespace, componentName));
     }
 
+    @Override
     public void restartFunctionInstances(final String tenant,
                                          final String namespace,
                                          final String componentName,
-                                         final String clientRole,
-                                         final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                         final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to restart {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "restart",
+                authParams);
 
         // validate parameters
         try {
@@ -873,26 +839,18 @@ public void restartFunctionInstances(final String tenant,
         }
     }
 
+    @Override
     public FunctionStatsImpl getFunctionStats(final String tenant,
                                               final String namespace,
                                               final String componentName,
                                               final URI uri,
-                                              final String clientRole,
-                                              final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                              final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to get stats for {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get stats for",
+                authParams);
 
         // validate parameters
         try {
@@ -941,22 +899,13 @@ public FunctionStatsImpl getFunctionStats(final String tenant,
                            final String componentName,
                            final String instanceId,
                            final URI uri,
-                           final String clientRole,
-                           final AuthenticationDataSource clientAuthenticationDataHttps) {
+                           final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to get stats for {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get stats for",
+                authParams);
 
         // validate parameters
         try {
@@ -1012,23 +961,13 @@ public FunctionStatsImpl getFunctionStats(final String tenant,
     @Override
     public List listFunctions(final String tenant,
                                       final String namespace,
-                                      final String clientRole,
-                                      final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                      final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{} Client [{}] is not authorized to list {}", tenant, namespace, clientRole,
-                        ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{} Failed to authorize [{}]", tenant, namespace, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, null, "list", authParams);
 
         // validate parameters
         try {
@@ -1070,13 +1009,13 @@ public List getListOfConnectors() {
     }
 
     @Override
-    public void reloadConnectors(String clientRole, AuthenticationDataSource authenticationData) {
+    public void reloadConnectors(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
         if (worker().getWorkerConfig().isAuthorizationEnabled()) {
             // Only superuser has permission to do this operation.
-            if (!isSuperUser(clientRole, authenticationData)) {
+            if (!isSuperUser(authParams)) {
                 throw new RestException(Status.UNAUTHORIZED, "This operation requires super-user access");
             }
         }
@@ -1094,23 +1033,13 @@ public String triggerFunction(final String tenant,
                                   final String input,
                                   final InputStream uploadedInputStream,
                                   final String topic,
-                                  final String clientRole,
-                                  final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                  final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to trigger {}", tenant, namespace,
-                        functionName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "trigger", authParams);
 
         // validate parameters
         try {
@@ -1224,23 +1153,14 @@ public FunctionState getFunctionState(final String tenant,
                                           final String namespace,
                                           final String functionName,
                                           final String key,
-                                          final String clientRole,
-                                          final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                          final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to get state for {}", tenant, namespace,
-                        functionName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "get state for",
+                authParams);
 
         if (null == worker().getStateStoreAdminClient()) {
             throwStateStoreUnvailableResponse();
@@ -1310,8 +1230,7 @@ public void putFunctionState(final String tenant,
                                  final String functionName,
                                  final String key,
                                  final FunctionState state,
-                                 final String clientRole,
-                                 final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                 final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
@@ -1321,16 +1240,8 @@ public void putFunctionState(final String tenant,
             throwStateStoreUnvailableResponse();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to put state for {}", tenant, namespace,
-                        functionName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "put state for",
+                authParams);
 
         if (!key.equals(state.getKey())) {
             log.error("{}/{}/{} Bad putFunction Request, path key doesn't match key in json", tenant, namespace,
@@ -1386,14 +1297,14 @@ public void putFunctionState(final String tenant,
     }
 
     @Override
-    public void uploadFunction(final InputStream uploadedInputStream, final String path, String clientRole,
-                               AuthenticationDataSource authenticationData) {
+    public void uploadFunction(final InputStream uploadedInputStream, final String path,
+                               AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole, authenticationData)) {
+        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(authParams)) {
             throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
         }
 
@@ -1428,23 +1339,13 @@ public void uploadFunction(final InputStream uploadedInputStream, final String p
 
     @Override
     public StreamingOutput downloadFunction(String tenant, String namespace, String componentName,
-                                            String clientRole, AuthenticationDataSource clientAuthenticationDataHttps,
-                                            boolean transformFunction) {
+                                            AuthenticationParameters authParams, boolean transformFunction) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not admin and authorized to download package for {} ", tenant,
-                        namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "download package for",
+                authParams);
 
         FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
         if (!functionMetaDataManager.containsFunction(tenant, namespace, componentName)) {
@@ -1529,8 +1430,7 @@ private Path getBuiltinArchivePath(String pkgPath, FunctionDetails.ComponentType
     }
 
     @Override
-    public StreamingOutput downloadFunction(
-            final String path, String clientRole, AuthenticationDataSource clientAuthenticationDataHttps) {
+    public StreamingOutput downloadFunction(final String path, final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
@@ -1544,19 +1444,10 @@ public StreamingOutput downloadFunction(
                 String namespace = tokens[1];
                 String componentName = tokens[2];
 
-                try {
-                    if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                        log.warn("{}/{}/{} Client [{}] is not admin and authorized to download package for {} ", tenant,
-                                namespace,
-                                componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                        throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-                    }
-                } catch (PulsarAdminException e) {
-                    log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-                    throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-                }
+                throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "download package for",
+                        authParams);
             } else {
-                if (!isSuperUser(clientRole, clientAuthenticationDataHttps)) {
+                if (!isSuperUser(authParams)) {
                     throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
                 }
             }
@@ -1694,57 +1585,61 @@ public static String createPackagePath(String tenant, String namespace, String f
                 getUniquePackageName(Codec.encode(fileName)));
     }
 
+    /**
+     * @deprecated use {@link #isAuthorizedRole(String, String, AuthenticationParameters)} instead.
+     */
+    @Deprecated
     public boolean isAuthorizedRole(String tenant, String namespace, String clientRole,
                                     AuthenticationDataSource authenticationData) throws PulsarAdminException {
-        if (worker().getWorkerConfig().isAuthorizationEnabled()) {
-            // skip authorization if client role is super-user
-            if (isSuperUser(clientRole, authenticationData)) {
-                return true;
-            }
-
-            if (clientRole != null) {
-                try {
-                    TenantInfoImpl tenantInfo =
-                            (TenantInfoImpl) worker().getBrokerAdmin().tenants().getTenantInfo(tenant);
-                    if (tenantInfo != null && worker().getAuthorizationService()
-                            .isTenantAdmin(tenant, clientRole, tenantInfo, authenticationData).get()) {
-                        return true;
-                    }
-                } catch (PulsarAdminException.NotFoundException | InterruptedException | ExecutionException e) {
+        AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole)
+                .clientAuthenticationDataSource(authenticationData).build();
+        return isAuthorizedRole(tenant, namespace, authParams);
+    }
 
-                }
-            }
+    public boolean isAuthorizedRole(String tenant, String namespace,
+                                    AuthenticationParameters authParams) throws PulsarAdminException {
+        if (worker().getWorkerConfig().isAuthorizationEnabled()) {
+            return allowFunctionOps(NamespaceName.get(tenant, namespace), authParams);
+        } else {
+            return true;
+        }
+    }
 
-            // check if role has permissions granted
-            if (clientRole != null && authenticationData != null) {
-                return allowFunctionOps(NamespaceName.get(tenant, namespace), clientRole, authenticationData);
-            } else {
-                return false;
+    public void throwRestExceptionIfUnauthorizedForNamespace(String tenant, String namespace, String componentName,
+                                                             String action, AuthenticationParameters authParams) {
+        try {
+            if (!isAuthorizedRole(tenant, namespace, authParams)) {
+                log.warn("{}/{}/{} Client with role [{}] and originalPrincipal [{}] is not authorized to {} {}",
+                        tenant, namespace, componentName, authParams.getClientRole(),
+                        authParams.getOriginalPrincipal(), action, ComponentTypeUtils.toString(componentType));
+                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
             }
+        } catch (PulsarAdminException e) {
+            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
+            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
         }
-        return true;
     }
 
-
+    @Deprecated
     protected void componentStatusRequestValidate(final String tenant, final String namespace,
                                                   final String componentName,
                                                   final String clientRole,
                                                   final AuthenticationDataSource clientAuthenticationDataHttps) {
+        AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole)
+                .clientAuthenticationDataSource(clientAuthenticationDataHttps).build();
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
+    }
+
+    protected void componentStatusRequestValidate(final String tenant, final String namespace,
+                                                  final String componentName,
+                                                  final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throw new RestException(Status.SERVICE_UNAVAILABLE,
                     "Function worker service is not done initializing. Please try again in a little while.");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized get status for {}", tenant, namespace,
-                        componentName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, componentName, e);
-            throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, componentName, "get status for",
+                authParams);
 
         // validate parameters
         try {
@@ -1773,13 +1668,25 @@ protected void componentStatusRequestValidate(final String tenant, final String
         }
     }
 
+    @Deprecated
     protected void componentInstanceStatusRequestValidate(final String tenant,
                                                       final String namespace,
                                                       final String componentName,
                                                       final int instanceId,
                                                       final String clientRole,
                                                       final AuthenticationDataSource clientAuthenticationDataHttps) {
-        componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps);
+        AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole)
+                .clientAuthenticationDataSource(clientAuthenticationDataHttps).build();
+        componentInstanceStatusRequestValidate(tenant, namespace, componentName, instanceId, authParams);
+    }
+
+    protected void componentInstanceStatusRequestValidate(final String tenant,
+                                                          final String namespace,
+                                                          final String componentName,
+                                                          final int instanceId,
+                                                          final AuthenticationParameters authParams) {
+
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
 
         FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
         FunctionMetaData functionMetaData =
@@ -1794,44 +1701,58 @@ protected void componentInstanceStatusRequestValidate(final String tenant,
         }
     }
 
-    public boolean isSuperUser(String clientRole, AuthenticationDataSource authenticationData) {
-        if (clientRole != null) {
+    public boolean isSuperUser(AuthenticationParameters authParams) {
+        if (authParams.getClientRole() != null) {
             try {
-                if ((worker().getWorkerConfig().getSuperUserRoles() != null
-                    && worker().getWorkerConfig().getSuperUserRoles().contains(clientRole))) {
-                    return true;
-                }
-                return worker().getAuthorizationService().isSuperUser(clientRole, authenticationData)
-                    .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+                return worker().getAuthorizationService().isSuperUser(authParams)
+                        .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
             } catch (InterruptedException e) {
-                log.warn("Time-out {} sec while checking the role {} is a super user role ",
-                    worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), clientRole);
+                log.warn("Time-out {} sec while checking the role {} originalPrincipal {} is a super user role ",
+                        worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(),
+                        authParams.getClientRole(), authParams.getOriginalPrincipal());
                 throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
             } catch (Exception e) {
-                log.warn("Admin-client with Role - failed to check the role {} is a super user role {} ", clientRole,
-                    e.getMessage(), e);
+                log.warn("Failed verifying role {} originalPrincipal {} is a super user role",
+                        authParams.getClientRole(), authParams.getOriginalPrincipal(), e);
                 throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
             }
         }
         return false;
     }
 
+    /**
+     * @deprecated use {@link #isSuperUser(AuthenticationParameters)}
+     */
+    @Deprecated
+    public boolean isSuperUser(String clientRole, AuthenticationDataSource authenticationData) {
+        AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(clientRole)
+                .clientAuthenticationDataSource(authenticationData).build();
+        return isSuperUser(authParams);
+    }
+
+    /**
+     * @deprecated use {@link #isSuperUser(AuthenticationParameters)}
+     */
+    @Deprecated
     public boolean allowFunctionOps(NamespaceName namespaceName, String role,
                                     AuthenticationDataSource authenticationData) {
+        AuthenticationParameters authParams = AuthenticationParameters.builder().clientRole(role)
+                .clientAuthenticationDataSource(authenticationData).build();
+        return allowFunctionOps(namespaceName, authParams);
+    }
+
+    public boolean allowFunctionOps(NamespaceName namespaceName, AuthenticationParameters authParams) {
         try {
             switch (componentType) {
                 case SINK:
-                    return worker().getAuthorizationService().allowSinkOpsAsync(
-                            namespaceName, role, authenticationData)
+                    return worker().getAuthorizationService().allowSinkOpsAsync(namespaceName, authParams)
                             .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
                 case SOURCE:
-                    return worker().getAuthorizationService().allowSourceOpsAsync(
-                            namespaceName, role, authenticationData)
+                    return worker().getAuthorizationService().allowSourceOpsAsync(namespaceName, authParams)
                             .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
                 case FUNCTION:
                 default:
-                    return worker().getAuthorizationService().allowFunctionOpsAsync(
-                            namespaceName, role, authenticationData)
+                    return worker().getAuthorizationService().allowFunctionOpsAsync(namespaceName, authParams)
                             .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
             }
         } catch (InterruptedException e) {
@@ -1839,9 +1760,9 @@ public boolean allowFunctionOps(NamespaceName namespaceName, String role,
                     worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), namespaceName);
             throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
         } catch (Exception e) {
-            log.warn("Admin-client with Role - {} failed to get function permissions for namespace - {}. {}", role,
-                    namespaceName,
-                    e.getMessage(), e);
+            log.warn("Admin-client with Role [{}] originalPrincipal [{}] failed to get function permissions for "
+                            + "namespace - {}. {}", authParams.getClientRole(),
+                    authParams.getOriginalPrincipal(), namespaceName, e.getMessage(), e);
             throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
         }
     }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index b7883e14c91e8..c7967600da5f6 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -38,7 +38,7 @@
 import javax.ws.rs.core.UriBuilder;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.FunctionDefinition;
@@ -78,8 +78,7 @@ public void registerFunction(final String tenant,
                                  final FormDataContentDisposition fileDetail,
                                  final String functionPkgUrl,
                                  final FunctionConfig functionConfig,
-                                 final String clientRole,
-                                 AuthenticationDataSource clientAuthenticationDataHttps) {
+                                 final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
@@ -98,16 +97,7 @@ public void registerFunction(final String tenant,
             throw new RestException(Response.Status.BAD_REQUEST, "Function config is not provided");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.error("{}/{}/{} Client [{}] is not authorized to register {}", tenant, namespace,
-                        functionName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e);
-            throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "register", authParams);
 
         try {
             // Check tenant exists
@@ -124,8 +114,8 @@ public void registerFunction(final String tenant,
                 }
             }
         } catch (PulsarAdminException.NotAuthorizedException e) {
-            log.error("{}/{}/{} Client [{}] is not authorized to operate {} on tenant", tenant, namespace,
-                    functionName, clientRole, ComponentTypeUtils.toString(componentType));
+            log.error("{}/{}/{} Client is not authorized to operate {} on tenant", tenant, namespace,
+                    functionName, ComponentTypeUtils.toString(componentType));
             throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
         } catch (PulsarAdminException.NotFoundException e) {
             log.error("{}/{}/{} Tenant {} does not exist", tenant, namespace, functionName, tenant);
@@ -193,11 +183,12 @@ public void registerFunction(final String tenant,
                 worker().getFunctionRuntimeManager()
                         .getRuntimeFactory()
                         .getAuthProvider().ifPresent(functionAuthProvider -> {
-                    if (clientAuthenticationDataHttps != null) {
+                    if (authParams.getClientAuthenticationDataSource() != null) {
 
                         try {
                             Optional functionAuthData = functionAuthProvider
-                                    .cacheAuthData(finalFunctionDetails, clientAuthenticationDataHttps);
+                                    .cacheAuthData(finalFunctionDetails,
+                                            authParams.getClientAuthenticationDataSource());
 
                             functionAuthData.ifPresent(authData -> functionMetaDataBuilder.setFunctionAuthSpec(
                                     Function.FunctionAuthenticationSpec.newBuilder()
@@ -245,8 +236,7 @@ public void updateFunction(final String tenant,
                                final FormDataContentDisposition fileDetail,
                                final String functionPkgUrl,
                                final FunctionConfig functionConfig,
-                               final String clientRole,
-                               AuthenticationDataSource clientAuthenticationDataHttps,
+                               final AuthenticationParameters authParams,
                                UpdateOptionsImpl updateOptions) {
 
         if (!isWorkerServiceAvailable()) {
@@ -266,17 +256,8 @@ public void updateFunction(final String tenant,
             throw new RestException(Response.Status.BAD_REQUEST, "Function config is not provided");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.error("{}/{}/{} Client [{}] is not authorized to update {}", tenant, namespace,
-                        functionName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, functionName, e);
-            throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, functionName, "update",
+                authParams);
 
         FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
 
@@ -357,7 +338,7 @@ public void updateFunction(final String tenant,
                 worker().getFunctionRuntimeManager()
                         .getRuntimeFactory()
                         .getAuthProvider().ifPresent(functionAuthProvider -> {
-                    if (clientAuthenticationDataHttps != null && updateOptions
+                    if (authParams.getClientAuthenticationDataSource() != null && updateOptions
                             != null && updateOptions.isUpdateAuthData()) {
                         // get existing auth data if it exists
                         Optional existingFunctionAuthData = Optional.empty();
@@ -369,7 +350,7 @@ public void updateFunction(final String tenant,
                                 try {
                                     Optional newFunctionAuthData = functionAuthProvider
                                             .updateAuthData(finalFunctionDetails, existingFunctionAuthData,
-                                                    clientAuthenticationDataHttps);
+                                                    authParams.getClientAuthenticationDataSource());
 
                             if (newFunctionAuthData.isPresent()) {
                                 functionMetaDataBuilder.setFunctionAuthSpec(
@@ -597,12 +578,11 @@ public FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData getFunct
             final String componentName,
             final String instanceId,
             final URI uri,
-            final String clientRole,
-            final AuthenticationDataSource clientAuthenticationDataHttps) {
+            final AuthenticationParameters authParams) {
 
         // validate parameters
         componentInstanceStatusRequestValidate(tenant, namespace, componentName, Integer.parseInt(instanceId),
-                clientRole, clientAuthenticationDataHttps);
+                authParams);
 
         FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData functionInstanceStatusData;
         try {
@@ -632,11 +612,10 @@ public FunctionStatus getFunctionStatus(final String tenant,
                                             final String namespace,
                                             final String componentName,
                                             final URI uri,
-                                            final String clientRole,
-                                            final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                            final AuthenticationParameters authParams) {
 
         // validate parameters
-        componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps);
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
 
         FunctionStatus functionStatus;
         try {
@@ -658,17 +637,18 @@ public void updateFunctionOnWorkerLeader(final String tenant,
                                              final InputStream uploadedInputStream,
                                              final boolean delete,
                                              URI uri,
-                                             final String clientRole,
-                                             AuthenticationDataSource authenticationData) {
+                                             final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
         if (worker().getWorkerConfig().isAuthorizationEnabled()) {
-            if (!isSuperUser(clientRole, authenticationData)) {
-                log.error("{}/{}/{} Client [{}] is not superuser to update on worker leader {}", tenant, namespace,
-                        functionName, clientRole, ComponentTypeUtils.toString(componentType));
+            if (!isSuperUser(authParams)) {
+                log.error("{}/{}/{} Client with role [{}] and originalPrincipal [{}] is not superuser to update on"
+                                + " worker leader {}", tenant, namespace, functionName, authParams.getClientRole(),
+                        authParams.getClientAuthenticationDataSource(),
+                        ComponentTypeUtils.toString(componentType));
                 throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
             }
         }
@@ -713,26 +693,26 @@ public void updateFunctionOnWorkerLeader(final String tenant,
     }
 
     @Override
-    public void reloadBuiltinFunctions(String clientRole, AuthenticationDataSource authenticationData)
+    public void reloadBuiltinFunctions(AuthenticationParameters authParams)
         throws IOException {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole, authenticationData)) {
+        if (worker().getWorkerConfig().isAuthorizationEnabled()
+                && !isSuperUser(authParams)) {
             throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
         }
         worker().getFunctionsManager().reloadFunctions(worker().getWorkerConfig());
     }
 
     @Override
-    public List getBuiltinFunctions(String clientRole,
-                                                        AuthenticationDataSource authenticationData) {
+    public List getBuiltinFunctions(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole, authenticationData)) {
+        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(authParams)) {
             throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
         }
         return this.worker().getFunctionsManager().getFunctionDefinitions();
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java
index 7b7f84aa82823..059db75542d59 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplV2.java
@@ -28,6 +28,7 @@
 import java.util.stream.Collectors;
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.FunctionState;
 import org.apache.pulsar.common.io.ConnectorDefinition;
@@ -59,11 +60,11 @@ public FunctionsImplV2(FunctionsImpl delegate) {
 
     @Override
     public Response getFunctionInfo(final String tenant, final String namespace,
-                                    final String functionName, String clientRole)
+                                    final String functionName, AuthenticationParameters authParams)
             throws IOException {
 
         // run just for parameter checks
-        delegate.getFunctionInfo(tenant, namespace, functionName, clientRole, null);
+        delegate.getFunctionInfo(tenant, namespace, functionName, authParams);
 
         FunctionMetaDataManager functionMetaDataManager = delegate.worker().getFunctionMetaDataManager();
 
@@ -75,11 +76,12 @@ public Response getFunctionInfo(final String tenant, final String namespace,
 
     @Override
     public Response getFunctionInstanceStatus(final String tenant, final String namespace, final String functionName,
-                                              final String instanceId, URI uri, String clientRole) throws IOException {
+                                              final String instanceId, URI uri,
+                                              AuthenticationParameters authParams) throws IOException {
 
         org.apache.pulsar.common.policies.data.FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData
                 functionInstanceStatus = delegate.getFunctionInstanceStatus(tenant, namespace,
-                functionName, instanceId, uri, clientRole, null);
+                functionName, instanceId, uri, authParams);
 
         String jsonResponse = FunctionCommon.printJson(toProto(functionInstanceStatus, instanceId));
         return Response.status(Response.Status.OK).entity(jsonResponse).build();
@@ -87,10 +89,10 @@ public Response getFunctionInstanceStatus(final String tenant, final String name
 
     @Override
     public Response getFunctionStatusV2(String tenant, String namespace, String functionName,
-                                        URI requestUri, String clientRole) throws
+                                        URI requestUri, AuthenticationParameters authParams) throws
             IOException {
         FunctionStatus functionStatus = delegate.getFunctionStatus(tenant, namespace,
-                functionName, requestUri, clientRole, null);
+                functionName, requestUri, authParams);
         InstanceCommunication.FunctionStatusList.Builder functionStatusList =
                 InstanceCommunication.FunctionStatusList.newBuilder();
         functionStatus.instances.forEach(functionInstanceStatus -> functionStatusList.addFunctionStatusList(
@@ -103,7 +105,7 @@ public Response getFunctionStatusV2(String tenant, String namespace, String func
     @Override
     public Response registerFunction(String tenant, String namespace, String functionName, InputStream
             uploadedInputStream, FormDataContentDisposition fileDetail, String functionPkgUrl, String
-                                             functionDetailsJson, String clientRole) {
+                                             functionDetailsJson, AuthenticationParameters authParams) {
 
         Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder();
         try {
@@ -114,14 +116,15 @@ public Response registerFunction(String tenant, String namespace, String functio
         FunctionConfig functionConfig = FunctionConfigUtils.convertFromDetails(functionDetailsBuilder.build());
 
         delegate.registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionConfig, clientRole, null);
+                functionPkgUrl, functionConfig, authParams);
         return Response.ok().build();
     }
 
     @Override
     public Response updateFunction(String tenant, String namespace, String functionName,
                                    InputStream uploadedInputStream, FormDataContentDisposition fileDetail,
-                                   String functionPkgUrl, String functionDetailsJson, String clientRole) {
+                                   String functionPkgUrl, String functionDetailsJson,
+                                   AuthenticationParameters authParams) {
 
         Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder();
         try {
@@ -132,35 +135,36 @@ public Response updateFunction(String tenant, String namespace, String functionN
         FunctionConfig functionConfig = FunctionConfigUtils.convertFromDetails(functionDetailsBuilder.build());
 
         delegate.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionConfig, clientRole, null, null);
+                functionPkgUrl, functionConfig, authParams, null);
         return Response.ok().build();
     }
 
     @Override
-    public Response deregisterFunction(String tenant, String namespace, String functionName, String clientAppId) {
-        delegate.deregisterFunction(tenant, namespace, functionName, clientAppId, null);
+    public Response deregisterFunction(String tenant, String namespace, String functionName,
+                                       AuthenticationParameters authParams) {
+        delegate.deregisterFunction(tenant, namespace, functionName, authParams);
         return Response.ok().build();
     }
 
     @Override
-    public Response listFunctions(String tenant, String namespace, String clientRole) {
-        Collection functionStateList = delegate.listFunctions(tenant, namespace, clientRole, null);
+    public Response listFunctions(String tenant, String namespace, AuthenticationParameters authParams) {
+        Collection functionStateList = delegate.listFunctions(tenant, namespace, authParams);
         return Response.status(Response.Status.OK).entity(new Gson().toJson(functionStateList.toArray())).build();
     }
 
     @Override
     public Response triggerFunction(String tenant, String namespace, String functionName, String triggerValue,
-                                    InputStream triggerStream, String topic, String clientRole) {
+                                    InputStream triggerStream, String topic, AuthenticationParameters authParams) {
         String result = delegate.triggerFunction(tenant, namespace, functionName,
-                triggerValue, triggerStream, topic, clientRole, null);
+                triggerValue, triggerStream, topic, authParams);
         return Response.status(Response.Status.OK).entity(result).build();
     }
 
     @Override
     public Response getFunctionState(String tenant, String namespace, String functionName,
-                                     String key, String clientRole) {
+                                     String key, AuthenticationParameters authParams) {
         FunctionState functionState = delegate.getFunctionState(
-                tenant, namespace, functionName, key, clientRole, null);
+                tenant, namespace, functionName, key, authParams);
 
         String value;
         if (functionState.getNumberValue() != null) {
@@ -175,39 +179,41 @@ public Response getFunctionState(String tenant, String namespace, String functio
 
     @Override
     public Response restartFunctionInstance(String tenant, String namespace, String functionName, String instanceId, URI
-            uri, String clientRole) {
-        delegate.restartFunctionInstance(tenant, namespace, functionName, instanceId, uri, clientRole, null);
+            uri, AuthenticationParameters authParams) {
+        delegate.restartFunctionInstance(tenant, namespace, functionName, instanceId, uri, authParams);
         return Response.ok().build();
     }
 
     @Override
-    public Response restartFunctionInstances(String tenant, String namespace, String functionName, String clientRole) {
-        delegate.restartFunctionInstances(tenant, namespace, functionName, clientRole, null);
+    public Response restartFunctionInstances(String tenant, String namespace, String functionName,
+                                             AuthenticationParameters authParams) {
+        delegate.restartFunctionInstances(tenant, namespace, functionName, authParams);
         return Response.ok().build();
     }
 
     @Override
     public Response stopFunctionInstance(String tenant, String namespace, String functionName, String instanceId, URI
-            uri, String clientRole) {
-        delegate.stopFunctionInstance(tenant, namespace, functionName, instanceId, uri, clientRole, null);
+            uri, AuthenticationParameters authParams) {
+        delegate.stopFunctionInstance(tenant, namespace, functionName, instanceId, uri, authParams);
         return Response.ok().build();
     }
 
     @Override
-    public Response stopFunctionInstances(String tenant, String namespace, String functionName, String clientRole) {
-        delegate.stopFunctionInstances(tenant, namespace, functionName, clientRole, null);
+    public Response stopFunctionInstances(String tenant, String namespace, String functionName,
+                                          AuthenticationParameters authParams) {
+        delegate.stopFunctionInstances(tenant, namespace, functionName, authParams);
         return Response.ok().build();
     }
 
     @Override
-    public Response uploadFunction(InputStream uploadedInputStream, String path, String clientRole) {
-        delegate.uploadFunction(uploadedInputStream, path, clientRole, null);
+    public Response uploadFunction(InputStream uploadedInputStream, String path, AuthenticationParameters authParams) {
+        delegate.uploadFunction(uploadedInputStream, path, authParams);
         return Response.ok().build();
     }
 
     @Override
-    public Response downloadFunction(String path, String clientRole) {
-        return Response.status(Response.Status.OK).entity(delegate.downloadFunction(path, clientRole, null)).build();
+    public Response downloadFunction(String path, AuthenticationParameters authParams) {
+        return Response.status(Response.Status.OK).entity(delegate.downloadFunction(path, authParams)).build();
     }
 
     @Override
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
index fff376c5dcca1..98450c4a0b5d0 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
@@ -38,7 +38,7 @@
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.common.functions.UpdateOptionsImpl;
 import org.apache.pulsar.common.functions.Utils;
@@ -77,8 +77,7 @@ public void registerSink(final String tenant,
                              final FormDataContentDisposition fileDetail,
                              final String sinkPkgUrl,
                              final SinkConfig sinkConfig,
-                             final String clientRole,
-                             AuthenticationDataSource clientAuthenticationDataHttps) {
+                             final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
@@ -97,16 +96,7 @@ public void registerSink(final String tenant,
             throw new RestException(Response.Status.BAD_REQUEST, "Sink config is not provided");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to register {}", tenant, namespace,
-                        sinkName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sinkName, e);
-            throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sinkName, "register", authParams);
 
         try {
             // Check tenant exists
@@ -123,8 +113,8 @@ public void registerSink(final String tenant,
                 }
             }
         } catch (PulsarAdminException.NotAuthorizedException e) {
-            log.error("{}/{}/{} Client [{}] is not authorized to operate {} on tenant", tenant, namespace,
-                    sinkName, clientRole, ComponentTypeUtils.toString(componentType));
+            log.error("{}/{}/{} Client is not authorized to operate {} on tenant", tenant, namespace,
+                    sinkName, ComponentTypeUtils.toString(componentType));
             throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
         } catch (PulsarAdminException.NotFoundException e) {
             log.error("{}/{}/{} Tenant {} does not exist", tenant, namespace, sinkName, tenant);
@@ -193,11 +183,12 @@ public void registerSink(final String tenant,
                 worker().getFunctionRuntimeManager()
                         .getRuntimeFactory()
                         .getAuthProvider().ifPresent(functionAuthProvider -> {
-                    if (clientAuthenticationDataHttps != null) {
+                    if (authParams.getClientAuthenticationDataSource() != null) {
 
                         try {
                             Optional functionAuthData = functionAuthProvider
-                                    .cacheAuthData(finalFunctionDetails, clientAuthenticationDataHttps);
+                                    .cacheAuthData(finalFunctionDetails,
+                                            authParams.getClientAuthenticationDataSource());
 
                             functionAuthData.ifPresent(authData -> functionMetaDataBuilder.setFunctionAuthSpec(
                                     Function.FunctionAuthenticationSpec.newBuilder()
@@ -251,8 +242,7 @@ public void updateSink(final String tenant,
                            final FormDataContentDisposition fileDetail,
                            final String sinkPkgUrl,
                            final SinkConfig sinkConfig,
-                           final String clientRole,
-                           AuthenticationDataSource clientAuthenticationDataHttps,
+                           final AuthenticationParameters authParams,
                            UpdateOptionsImpl updateOptions) {
 
         if (!isWorkerServiceAvailable()) {
@@ -272,17 +262,7 @@ public void updateSink(final String tenant,
             throw new RestException(Response.Status.BAD_REQUEST, "Sink config is not provided");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to update {}", tenant, namespace,
-                        sinkName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sinkName, e);
-            throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sinkName, "update", authParams);
 
         FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
 
@@ -363,7 +343,7 @@ public void updateSink(final String tenant,
                 worker().getFunctionRuntimeManager()
                         .getRuntimeFactory()
                         .getAuthProvider().ifPresent(functionAuthProvider -> {
-                    if (clientAuthenticationDataHttps != null && updateOptions != null
+                    if (authParams.getClientAuthenticationDataSource() != null && updateOptions != null
                             && updateOptions.isUpdateAuthData()) {
                         // get existing auth data if it exists
                         Optional existingFunctionAuthData = Optional.empty();
@@ -375,7 +355,7 @@ public void updateSink(final String tenant,
                         try {
                             Optional newFunctionAuthData = functionAuthProvider
                                     .updateAuthData(finalFunctionDetails, existingFunctionAuthData,
-                                            clientAuthenticationDataHttps);
+                                            authParams.getClientAuthenticationDataSource());
 
                             if (newFunctionAuthData.isPresent()) {
                                 functionMetaDataBuilder.setFunctionAuthSpec(
@@ -629,12 +609,11 @@ private ExceptionInformation getExceptionInformation(InstanceCommunication.Funct
                           final String sinkName,
                           final String instanceId,
                           final URI uri,
-                          final String clientRole,
-                          final AuthenticationDataSource clientAuthenticationDataHttps) {
+                          final AuthenticationParameters authParams) {
 
         // validate parameters
-        componentInstanceStatusRequestValidate(tenant, namespace, sinkName, Integer.parseInt(instanceId), clientRole,
-                clientAuthenticationDataHttps);
+        componentInstanceStatusRequestValidate(tenant, namespace, sinkName, Integer.parseInt(instanceId),
+                authParams);
 
 
         SinkStatus.SinkInstanceStatus.SinkInstanceStatusData sinkInstanceStatusData;
@@ -655,11 +634,10 @@ public SinkStatus getSinkStatus(final String tenant,
                                     final String namespace,
                                     final String componentName,
                                     final URI uri,
-                                    final String clientRole,
-                                    final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                    final AuthenticationParameters authParams) {
 
         // validate parameters
-        componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps);
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
 
         SinkStatus sinkStatus;
         try {
@@ -677,38 +655,12 @@ public SinkStatus getSinkStatus(final String tenant,
     @Override
     public SinkConfig getSinkInfo(final String tenant,
                                   final String namespace,
-                                  final String componentName) {
-
-        if (!isWorkerServiceAvailable()) {
-            throwUnavailableException();
-        }
-
-        // validate parameters
-        try {
-            validateGetFunctionRequestParams(tenant, namespace, componentName, componentType);
-        } catch (IllegalArgumentException e) {
-            log.error("Invalid get {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant,
-                    namespace, componentName, e);
-            throw new RestException(Response.Status.BAD_REQUEST, e.getMessage());
-        }
-
-        FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
-        if (!functionMetaDataManager.containsFunction(tenant, namespace, componentName)) {
-            log.error("{} does not exist @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, namespace,
-                    componentName);
-            throw new RestException(Response.Status.NOT_FOUND,
-                    String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName));
-        }
+                                  final String componentName,
+                                  final AuthenticationParameters authParams) {
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
         Function.FunctionMetaData functionMetaData =
-                functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName);
-        if (!InstanceUtils.calculateSubjectType(functionMetaData.getFunctionDetails()).equals(componentType)) {
-            log.error("{}/{}/{} is not a {}", tenant, namespace, componentName,
-                    ComponentTypeUtils.toString(componentType));
-            throw new RestException(Response.Status.NOT_FOUND,
-                    String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName));
-        }
-        SinkConfig config = SinkConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails());
-        return config;
+                worker().getFunctionMetaDataManager().getFunctionMetaData(tenant, namespace, componentName);
+        return SinkConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails());
     }
 
     @Override
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
index 2c5921bf7ea9d..876c7e7572e78 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
@@ -37,7 +37,7 @@
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.common.functions.UpdateOptionsImpl;
 import org.apache.pulsar.common.functions.Utils;
@@ -76,8 +76,7 @@ public void registerSource(final String tenant,
                                final FormDataContentDisposition fileDetail,
                                final String sourcePkgUrl,
                                final SourceConfig sourceConfig,
-                               final String clientRole,
-                               AuthenticationDataSource clientAuthenticationDataHttps) {
+                               final AuthenticationParameters authParams) {
 
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
@@ -96,16 +95,7 @@ public void registerSource(final String tenant,
             throw new RestException(Response.Status.BAD_REQUEST, "Source config is not provided");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to register {}", tenant, namespace,
-                        sourceName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sourceName, e);
-            throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sourceName, "register", authParams);
 
         try {
             // Check tenant exists
@@ -122,8 +112,8 @@ public void registerSource(final String tenant,
                 }
             }
         } catch (PulsarAdminException.NotAuthorizedException e) {
-            log.error("{}/{}/{} Client [{}] is not authorized to operate {} on tenant", tenant, namespace,
-                    sourceName, clientRole, ComponentTypeUtils.toString(componentType));
+            log.error("{}/{}/{} Client is not authorized to operate {} on tenant", tenant, namespace,
+                    sourceName, ComponentTypeUtils.toString(componentType));
             throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
         } catch (PulsarAdminException.NotFoundException e) {
             log.error("{}/{}/{} Tenant {} does not exist", tenant, namespace, sourceName, tenant);
@@ -193,11 +183,12 @@ public void registerSource(final String tenant,
                 worker().getFunctionRuntimeManager()
                         .getRuntimeFactory()
                         .getAuthProvider().ifPresent(functionAuthProvider -> {
-                    if (clientAuthenticationDataHttps != null) {
+                    if (authParams.getClientAuthenticationDataSource() != null) {
 
                         try {
                             Optional functionAuthData = functionAuthProvider
-                                    .cacheAuthData(finalFunctionDetails, clientAuthenticationDataHttps);
+                                    .cacheAuthData(finalFunctionDetails,
+                                            authParams.getClientAuthenticationDataSource());
 
                             functionAuthData.ifPresent(authData -> functionMetaDataBuilder.setFunctionAuthSpec(
                                     Function.FunctionAuthenticationSpec.newBuilder()
@@ -245,8 +236,7 @@ public void updateSource(final String tenant,
                                final FormDataContentDisposition fileDetail,
                                final String sourcePkgUrl,
                                final SourceConfig sourceConfig,
-                               final String clientRole,
-                               AuthenticationDataSource clientAuthenticationDataHttps,
+                               final AuthenticationParameters authParams,
                                UpdateOptionsImpl updateOptions) {
 
         if (!isWorkerServiceAvailable()) {
@@ -266,17 +256,7 @@ public void updateSource(final String tenant,
             throw new RestException(Response.Status.BAD_REQUEST, "Source config is not provided");
         }
 
-        try {
-            if (!isAuthorizedRole(tenant, namespace, clientRole, clientAuthenticationDataHttps)) {
-                log.warn("{}/{}/{} Client [{}] is not authorized to update {}", tenant, namespace,
-                        sourceName, clientRole, ComponentTypeUtils.toString(componentType));
-                throw new RestException(Response.Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-
-            }
-        } catch (PulsarAdminException e) {
-            log.error("{}/{}/{} Failed to authorize [{}]", tenant, namespace, sourceName, e);
-            throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
-        }
+        throwRestExceptionIfUnauthorizedForNamespace(tenant, namespace, sourceName, "update", authParams);
 
         FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
 
@@ -357,7 +337,7 @@ public void updateSource(final String tenant,
                 worker().getFunctionRuntimeManager()
                         .getRuntimeFactory()
                         .getAuthProvider().ifPresent(functionAuthProvider -> {
-                    if (clientAuthenticationDataHttps != null && updateOptions != null
+                    if (authParams.getClientAuthenticationDataSource() != null && updateOptions != null
                             && updateOptions.isUpdateAuthData()) {
                         // get existing auth data if it exists
                         Optional existingFunctionAuthData = Optional.empty();
@@ -369,7 +349,7 @@ public void updateSource(final String tenant,
                         try {
                             Optional newFunctionAuthData = functionAuthProvider
                                     .updateAuthData(finalFunctionDetails, existingFunctionAuthData,
-                                            clientAuthenticationDataHttps);
+                                            authParams.getClientAuthenticationDataSource());
 
                             if (newFunctionAuthData.isPresent()) {
                                 functionMetaDataBuilder.setFunctionAuthSpec(
@@ -591,10 +571,10 @@ public SourceStatus emptyStatus(final int parallelism) {
     public SourceStatus getSourceStatus(final String tenant,
                                         final String namespace,
                                         final String componentName,
-                                        final URI uri, final String clientRole,
-                                        final AuthenticationDataSource clientAuthenticationDataHttps) {
+                                        final URI uri,
+                                        final AuthenticationParameters authParams) {
         // validate parameters
-        componentStatusRequestValidate(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps);
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
 
         SourceStatus sourceStatus;
         try {
@@ -616,11 +596,9 @@ public SourceStatus getSourceStatus(final String tenant,
                             final String sourceName,
                             final String instanceId,
                             final URI uri,
-                            final String clientRole,
-                            final AuthenticationDataSource clientAuthenticationDataHttps) {
+                            final AuthenticationParameters authParams) {
         // validate parameters
-        componentInstanceStatusRequestValidate(tenant, namespace, sourceName, Integer.parseInt(instanceId), clientRole,
-                clientAuthenticationDataHttps);
+        componentInstanceStatusRequestValidate(tenant, namespace, sourceName, Integer.parseInt(instanceId), authParams);
 
         SourceStatus.SourceInstanceStatus.SourceInstanceStatusData sourceInstanceStatusData;
         try {
@@ -638,38 +616,12 @@ public SourceStatus getSourceStatus(final String tenant,
     @Override
     public SourceConfig getSourceInfo(final String tenant,
                                       final String namespace,
-                                      final String componentName) {
-
-        if (!isWorkerServiceAvailable()) {
-            throwUnavailableException();
-        }
-
-        // validate parameters
-        try {
-            validateGetFunctionRequestParams(tenant, namespace, componentName, componentType);
-        } catch (IllegalArgumentException e) {
-            log.error("Invalid get {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant,
-                    namespace, componentName, e);
-            throw new RestException(Response.Status.BAD_REQUEST, e.getMessage());
-        }
-
-        FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
-        if (!functionMetaDataManager.containsFunction(tenant, namespace, componentName)) {
-            log.error("{} does not exist @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, namespace,
-                    componentName);
-            throw new RestException(Response.Status.NOT_FOUND,
-                    String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName));
-        }
+                                      final String componentName,
+                                      final AuthenticationParameters authParams) {
+        componentStatusRequestValidate(tenant, namespace, componentName, authParams);
         Function.FunctionMetaData functionMetaData =
-                functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName);
-        if (!InstanceUtils.calculateSubjectType(functionMetaData.getFunctionDetails()).equals(componentType)) {
-            log.error("{}/{}/{} is not a {}", tenant, namespace, componentName,
-                    ComponentTypeUtils.toString(componentType));
-            throw new RestException(Response.Status.NOT_FOUND,
-                    String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName));
-        }
-        SourceConfig config = SourceConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails());
-        return config;
+                worker().getFunctionMetaDataManager().getFunctionMetaData(tenant, namespace, componentName);
+        return SourceConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails());
     }
 
     @Override
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java
index 364c3ec1ab104..0b77be4ab0212 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/WorkerImpl.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.functions.worker.rest.api;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException;
 import java.io.IOException;
 import java.net.URI;
@@ -27,12 +28,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
 import java.util.function.Supplier;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 import javax.ws.rs.core.UriBuilder;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.LongRunningProcessStatus;
 import org.apache.pulsar.common.functions.WorkerInfo;
 import org.apache.pulsar.common.io.ConnectorDefinition;
@@ -77,29 +81,24 @@ private boolean isWorkerServiceAvailable() {
     }
 
     @Override
-    public List getCluster(String clientRole) {
+    public List getCluster(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
+        throwIfNotSuperUser(authParams, "get cluster");
 
         List workers = worker().getMembershipManager().getCurrentMembership();
         return workers;
     }
 
     @Override
-    public WorkerInfo getClusterLeader(String clientRole) {
+    public WorkerInfo getClusterLeader(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to get cluster leader", clientRole);
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
+        throwIfNotSuperUser(authParams, "get cluster leader");
 
         MembershipManager membershipManager = worker().getMembershipManager();
         WorkerInfo leader = membershipManager.getLeader();
@@ -112,15 +111,12 @@ public WorkerInfo getClusterLeader(String clientRole) {
     }
 
     @Override
-    public Map> getAssignments(String clientRole) {
+    public Map> getAssignments(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to get cluster assignments", clientRole);
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
+        throwIfNotSuperUser(authParams, "get cluster assignments");
 
         FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager();
         Map> assignments = functionRuntimeManager.getCurrentAssignments();
@@ -131,33 +127,41 @@ public Map> getAssignments(String clientRole) {
         return ret;
     }
 
-    private boolean isSuperUser(final String clientRole) {
-        return clientRole != null && worker().getWorkerConfig().getSuperUserRoles().contains(clientRole);
+    private void throwIfNotSuperUser(AuthenticationParameters authParams, String action) {
+        if (worker().getWorkerConfig().isAuthorizationEnabled()) {
+            try {
+                if (authParams.getClientRole() == null || !worker().getAuthorizationService().isSuperUser(authParams)
+                        .get(worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS)) {
+                    log.error("Client with role [{}] and originalPrincipal [{}] is not authorized to {}",
+                            authParams.getClientRole(), authParams.getOriginalPrincipal(), action);
+                    throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
+                }
+            } catch (ExecutionException | TimeoutException | InterruptedException e) {
+                log.warn("Time-out {} sec while checking the role {} originalPrincipal {} is a super user role ",
+                        worker().getWorkerConfig().getMetadataStoreOperationTimeoutSeconds(),
+                        authParams.getClientRole(), authParams.getOriginalPrincipal());
+                throw new RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+            }
+        }
     }
 
     @Override
-    public List getWorkerMetrics(final String clientRole) {
+    public List getWorkerMetrics(final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable() || worker().getMetricsGenerator() == null) {
             throwUnavailableException();
         }
-
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to get worker stats", clientRole);
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
+        throwIfNotSuperUser(authParams, "get worker stats");
         return worker().getMetricsGenerator().generate();
     }
 
     @Override
-    public List getFunctionsMetrics(String clientRole) throws IOException {
+    public List getFunctionsMetrics(AuthenticationParameters authParams)
+            throws IOException {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to get function stats", clientRole);
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
+        throwIfNotSuperUser(authParams, "get function stats");
 
         Map functionRuntimes = worker().getFunctionRuntimeManager()
                 .getFunctionRuntimeInfos();
@@ -196,28 +200,20 @@ public List getFunctionsMetrics(String clientRole)
     }
 
     @Override
-    public List getListOfConnectors(String clientRole) {
+    public List getListOfConnectors(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
-
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
-
+        throwIfNotSuperUser(authParams, "get list of connectors");
         return this.worker().getConnectorsManager().getConnectorDefinitions();
     }
 
     @Override
-    public void rebalance(final URI uri, final String clientRole) {
+    public void rebalance(final URI uri, final AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
-
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to rebalance cluster", clientRole);
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
-        }
+        throwIfNotSuperUser(authParams, "rebalance cluster");
 
         if (worker().getLeaderService().isLeader()) {
             try {
@@ -239,7 +235,8 @@ public void rebalance(final URI uri, final String clientRole) {
     }
 
     @Override
-    public void drain(final URI uri, final String inWorkerId, final String clientRole, boolean calledOnLeaderUri) {
+    public void drain(final URI uri, final String inWorkerId, final AuthenticationParameters authParams,
+                      boolean calledOnLeaderUri) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
@@ -248,15 +245,13 @@ public void drain(final URI uri, final String inWorkerId, final String clientRol
         final String workerId = (inWorkerId == null || inWorkerId.isEmpty()) ? actualWorkerId : inWorkerId;
 
         if (log.isDebugEnabled()) {
-            log.debug("drain called with URI={}, inWorkerId={}, workerId={}, clientRole={}, calledOnLeaderUri={}, "
-                    + "on actual worker-id={}",
-                    uri, inWorkerId, workerId, clientRole, calledOnLeaderUri, actualWorkerId);
+            log.debug("drain called with URI={}, inWorkerId={}, workerId={}, clientRole={}, originalPrincipal={}, "
+                            + "calledOnLeaderUri={}, on actual worker-id={}",
+                    uri, inWorkerId, workerId, authParams.getClientRole(), authParams.getOriginalPrincipal(),
+                    calledOnLeaderUri, actualWorkerId);
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to drain worker {}", clientRole, workerId);
-            throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform drain operation");
-        }
+        throwIfNotSuperUser(authParams, "drain worker");
 
         // Depending on which operations we decide to allow, we may add checks here to error/exception if
         //      calledOnLeaderUri is true on a non-leader
@@ -285,7 +280,8 @@ public void drain(final URI uri, final String inWorkerId, final String clientRol
     }
 
     @Override
-    public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWorkerId, final String clientRole,
+    public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWorkerId,
+                                                   final AuthenticationParameters authParams,
                                                    boolean calledOnLeaderUri) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
@@ -296,15 +292,12 @@ public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWor
 
         if (log.isDebugEnabled()) {
             log.debug("getDrainStatus called with uri={}, inWorkerId={}, workerId={}, clientRole={}, "
-                            + " calledOnLeaderUri={}, on actual workerId={}",
-                    uri, inWorkerId, workerId, clientRole, calledOnLeaderUri, actualWorkerId);
+                            + "originalPrincipal={}, calledOnLeaderUri={}, on actual workerId={}",
+                    uri, inWorkerId, workerId, authParams.getClientRole(), authParams.getOriginalPrincipal(),
+                    calledOnLeaderUri, actualWorkerId);
         }
 
-        if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
-            log.error("Client [{}] is not authorized to get drain status of worker {}", clientRole, workerId);
-            throw new RestException(Status.UNAUTHORIZED,
-                    "Client is not authorized to get the status of a drain operation");
-        }
+        throwIfNotSuperUser(authParams, "get drain status of worker");
 
         // Depending on which operations we decide to allow, we may add checks here to error/exception if
         //      calledOnLeaderUri is true on a non-leader
@@ -321,7 +314,7 @@ public LongRunningProcessStatus getDrainStatus(final URI uri, final String inWor
     }
 
     @Override
-    public Boolean isLeaderReady(final String clientRole) {
+    public boolean isLeaderReady(AuthenticationParameters authParams) {
         if (!isWorkerServiceAvailable()) {
             throwUnavailableException();
         }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java
index 9176013b783f6..0b125756b30a1 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionsApiV2Resource.java
@@ -72,7 +72,7 @@ public Response registerFunction(final @PathParam("tenant") String tenant,
                                      final @FormDataParam("functionDetails") String functionDetailsJson) {
 
         return functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionDetailsJson, clientAppId());
+                functionPkgUrl, functionDetailsJson, authParams());
     }
 
     @PUT
@@ -93,7 +93,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant,
                                    final @FormDataParam("functionDetails") String functionDetailsJson) {
 
         return functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionDetailsJson, clientAppId());
+                functionPkgUrl, functionDetailsJson, authParams());
     }
 
 
@@ -110,7 +110,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant,
     public Response deregisterFunction(final @PathParam("tenant") String tenant,
                                        final @PathParam("namespace") String namespace,
                                        final @PathParam("functionName") String functionName) {
-        return functions().deregisterFunction(tenant, namespace, functionName, clientAppId());
+        return functions().deregisterFunction(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -129,8 +129,7 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace,
                                     final @PathParam("functionName") String functionName) throws IOException {
 
-        return functions().getFunctionInfo(
-                tenant, namespace, functionName, clientAppId());
+        return functions().getFunctionInfo(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -151,7 +150,7 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena
                                               final @PathParam("instanceId") String instanceId) throws IOException {
 
         return functions().getFunctionInstanceStatus(tenant, namespace, functionName, instanceId, uri.getRequestUri(),
-                clientAppId());
+                authParams());
     }
 
     @GET
@@ -168,7 +167,7 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena
     public Response getFunctionStatus(final @PathParam("tenant") String tenant,
                                       final @PathParam("namespace") String namespace,
                                       final @PathParam("functionName") String functionName) throws IOException {
-        return functions().getFunctionStatusV2(tenant, namespace, functionName, uri.getRequestUri(), clientAppId());
+        return functions().getFunctionStatusV2(tenant, namespace, functionName, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -184,7 +183,7 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant,
     @Path("/{tenant}/{namespace}")
     public Response listFunctions(final @PathParam("tenant") String tenant,
                                   final @PathParam("namespace") String namespace) {
-        return functions().listFunctions(tenant, namespace, clientAppId());
+        return functions().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -207,7 +206,7 @@ public Response triggerFunction(final @PathParam("tenant") String tenant,
                                     final @FormDataParam("dataStream") InputStream triggerStream,
                                     final @FormDataParam("topic") String topic) {
         return functions().triggerFunction(tenant, namespace, functionName, triggerValue, triggerStream, topic,
-                clientAppId());
+                authParams());
     }
 
     @GET
@@ -226,7 +225,7 @@ public Response getFunctionState(final @PathParam("tenant") String tenant,
                                      final @PathParam("namespace") String namespace,
                                      final @PathParam("functionName") String functionName,
                                      final @PathParam("key") String key) {
-        return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId());
+        return functions().getFunctionState(tenant, namespace, functionName, key, authParams());
     }
 
     @POST
@@ -243,7 +242,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant,
                                     final @PathParam("functionName") String functionName,
                                     final @PathParam("instanceId") String instanceId) {
         return functions().restartFunctionInstance(tenant, namespace, functionName, instanceId, uri.getRequestUri(),
-                clientAppId());
+                authParams());
     }
 
     @POST
@@ -256,7 +255,7 @@ public Response restartFunction(final @PathParam("tenant") String tenant,
     public Response restartFunction(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace,
                                     final @PathParam("functionName") String functionName) {
-        return functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId());
+        return functions().restartFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -271,7 +270,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant,
                                  final @PathParam("functionName") String functionName,
                                  final @PathParam("instanceId") String instanceId) {
         return functions().stopFunctionInstance(tenant, namespace, functionName, instanceId, uri.getRequestUri(),
-                clientAppId());
+                authParams());
     }
 
     @POST
@@ -284,7 +283,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant,
     public Response stopFunction(final @PathParam("tenant") String tenant,
                                  final @PathParam("namespace") String namespace,
                                  final @PathParam("functionName") String functionName) {
-        return functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId());
+        return functions().stopFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -296,7 +295,7 @@ public Response stopFunction(final @PathParam("tenant") String tenant,
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
                                    final @FormDataParam("path") String path) {
-        return functions().uploadFunction(uploadedInputStream, path, clientAppId());
+        return functions().uploadFunction(uploadedInputStream, path, authParams());
     }
 
     @GET
@@ -306,7 +305,7 @@ public Response uploadFunction(final @FormDataParam("data") InputStream uploaded
     )
     @Path("/download")
     public Response downloadFunction(final @QueryParam("path") String path) {
-        return functions().downloadFunction(path, clientAppId());
+        return functions().downloadFunction(path, authParams());
     }
 
     @GET
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java
index 496f6f43028fe..276d0ae86d876 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerApiV2Resource.java
@@ -39,11 +39,14 @@
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.web.AuthenticationFilter;
 import org.apache.pulsar.client.admin.LongRunningProcessStatus;
 import org.apache.pulsar.common.functions.WorkerInfo;
 import org.apache.pulsar.common.io.ConnectorDefinition;
 import org.apache.pulsar.functions.worker.WorkerService;
+import org.apache.pulsar.functions.worker.rest.FunctionApiResource;
 import org.apache.pulsar.functions.worker.service.api.Workers;
 
 @Slf4j
@@ -75,12 +78,25 @@ Workers workers() {
         return get().getWorkers();
     }
 
+    /**
+     * @deprecated use {@link #authParams()} instead
+     */
+    @Deprecated
     public String clientAppId() {
         return httpRequest != null
                 ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName)
                 : null;
     }
 
+    public AuthenticationParameters authParams() {
+        return AuthenticationParameters.builder()
+                .clientRole(clientAppId())
+                .originalPrincipal(httpRequest.getHeader(FunctionApiResource.ORIGINAL_PRINCIPAL_HEADER))
+                .clientAuthenticationDataSource((AuthenticationDataSource)
+                        httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName))
+                .build();
+    }
+
     @GET
     @ApiOperation(
             value = "Fetches information about the Pulsar cluster running Pulsar Functions",
@@ -94,7 +110,7 @@ public String clientAppId() {
     @Path("/cluster")
     @Produces(MediaType.APPLICATION_JSON)
     public List getCluster() {
-        return workers().getCluster(clientAppId());
+        return workers().getCluster(authParams());
     }
 
     @GET
@@ -109,7 +125,7 @@ public List getCluster() {
     @Path("/cluster/leader")
     @Produces(MediaType.APPLICATION_JSON)
     public WorkerInfo getClusterLeader() {
-        return workers().getClusterLeader(clientAppId());
+        return workers().getClusterLeader(authParams());
     }
 
     @GET
@@ -124,7 +140,7 @@ public WorkerInfo getClusterLeader() {
     @Path("/assignments")
     @Produces(MediaType.APPLICATION_JSON)
     public Map> getAssignments() {
-        return workers().getAssignments(clientAppId());
+        return workers().getAssignments(authParams());
     }
 
     @GET
@@ -139,7 +155,7 @@ public Map> getAssignments() {
     })
     @Path("/connectors")
     public List getConnectorsList() throws IOException {
-        return workers().getListOfConnectors(clientAppId());
+        return workers().getListOfConnectors(authParams());
     }
 
     @PUT
@@ -153,7 +169,7 @@ public List getConnectorsList() throws IOException {
     })
     @Path("/rebalance")
     public void rebalance() {
-        workers().rebalance(uri.getRequestUri(), clientAppId());
+        workers().rebalance(uri.getRequestUri(), authParams());
     }
 
     @PUT
@@ -169,7 +185,7 @@ public void rebalance() {
     })
     @Path("/leader/drain")
     public void drainAtLeader(@QueryParam("workerId") String workerId) {
-        workers().drain(uri.getRequestUri(), workerId, clientAppId(), true);
+        workers().drain(uri.getRequestUri(), workerId, authParams(), true);
     }
 
     @PUT
@@ -185,7 +201,7 @@ public void drainAtLeader(@QueryParam("workerId") String workerId) {
     })
     @Path("/drain")
     public void drain() {
-        workers().drain(uri.getRequestUri(), null, clientAppId(), false);
+        workers().drain(uri.getRequestUri(), null, authParams(), false);
     }
 
     @GET
@@ -199,7 +215,7 @@ public void drain() {
     })
     @Path("/leader/drain")
     public LongRunningProcessStatus getDrainStatus(@QueryParam("workerId") String workerId) {
-        return workers().getDrainStatus(uri.getRequestUri(), workerId, clientAppId(), true);
+        return workers().getDrainStatus(uri.getRequestUri(), workerId, authParams(), true);
     }
 
     @GET
@@ -213,7 +229,7 @@ public LongRunningProcessStatus getDrainStatus(@QueryParam("workerId") String wo
     })
     @Path("/drain")
     public LongRunningProcessStatus getDrainStatus() {
-        return workers().getDrainStatus(uri.getRequestUri(), null, clientAppId(), false);
+        return workers().getDrainStatus(uri.getRequestUri(), null, authParams(), false);
     }
 
     @GET
@@ -226,6 +242,6 @@ public LongRunningProcessStatus getDrainStatus() {
     })
     @Path("/cluster/leader/ready")
     public Boolean isLeaderReady() {
-        return workers().isLeaderReady(clientAppId());
+        return workers().isLeaderReady(authParams());
     }
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java
index b6a7914f0fdfd..712505a4ba08a 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/WorkerStatsApiV2Resource.java
@@ -34,9 +34,12 @@
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.web.AuthenticationFilter;
 import org.apache.pulsar.common.policies.data.WorkerFunctionInstanceStats;
 import org.apache.pulsar.functions.worker.WorkerService;
+import org.apache.pulsar.functions.worker.rest.FunctionApiResource;
 import org.apache.pulsar.functions.worker.service.api.Workers;
 
 @Slf4j
@@ -66,6 +69,19 @@ Workers workers() {
         return get().getWorkers();
     }
 
+    AuthenticationParameters authParams() {
+        return AuthenticationParameters.builder()
+                .clientRole(clientAppId())
+                .originalPrincipal(httpRequest.getHeader(FunctionApiResource.ORIGINAL_PRINCIPAL_HEADER))
+                .clientAuthenticationDataSource((AuthenticationDataSource)
+                        httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName))
+                .build();
+    }
+
+    /**
+     * @deprecated use {@link AuthenticationParameters} instead
+     */
+    @Deprecated
     public String clientAppId() {
         return httpRequest != null
                 ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName)
@@ -85,7 +101,7 @@ public String clientAppId() {
     })
     @Produces(MediaType.APPLICATION_JSON)
     public List getMetrics() throws Exception {
-        return workers().getWorkerMetrics(clientAppId());
+        return workers().getWorkerMetrics(authParams());
     }
 
     @GET
@@ -101,6 +117,6 @@ public List getMetrics() throws Exceptio
     })
     @Produces(MediaType.APPLICATION_JSON)
     public List getStats() throws IOException {
-        return workers().getFunctionsMetrics(clientAppId());
+        return workers().getFunctionsMetrics(authParams());
     }
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java
index bca0cf4bf3d75..7bdc86d5fae3e 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java
@@ -73,7 +73,7 @@ public void registerFunction(final @PathParam("tenant") String tenant,
                                  final @FormDataParam("functionConfig") FunctionConfig functionConfig) {
 
         functions().registerFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionConfig, clientAppId(), clientAuthData());
+                functionPkgUrl, functionConfig, authParams());
 
     }
 
@@ -90,7 +90,7 @@ public void updateFunction(final @PathParam("tenant") String tenant,
                                final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) {
 
         functions().updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail,
-                functionPkgUrl, functionConfig, clientAppId(), clientAuthData(), updateOptions);
+                functionPkgUrl, functionConfig, authParams(), updateOptions);
 
     }
 
@@ -99,7 +99,7 @@ public void updateFunction(final @PathParam("tenant") String tenant,
     public void deregisterFunction(final @PathParam("tenant") String tenant,
                                    final @PathParam("namespace") String namespace,
                                    final @PathParam("functionName") String functionName) {
-        functions().deregisterFunction(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().deregisterFunction(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -107,7 +107,7 @@ public void deregisterFunction(final @PathParam("tenant") String tenant,
     public FunctionConfig getFunctionInfo(final @PathParam("tenant") String tenant,
                                           final @PathParam("namespace") String namespace,
                                           final @PathParam("functionName") String functionName) {
-        return functions().getFunctionInfo(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        return functions().getFunctionInfo(tenant, namespace, functionName, authParams());
     }
 
     @GET
@@ -115,7 +115,7 @@ public FunctionConfig getFunctionInfo(final @PathParam("tenant") String tenant,
     @Path("/{tenant}/{namespace}")
     public List listFunctions(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace) {
-        return functions().listFunctions(tenant, namespace, clientAppId(), clientAuthData());
+        return functions().listFunctions(tenant, namespace, authParams());
     }
 
     @GET
@@ -137,7 +137,7 @@ public FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData getFunct
             final @PathParam("functionName") String functionName,
             final @PathParam("instanceId") String instanceId) throws IOException {
         return functions().getFunctionInstanceStatus(
-                tenant, namespace, functionName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+                tenant, namespace, functionName, instanceId, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -158,7 +158,7 @@ public FunctionStatus getFunctionStatus(
             final @PathParam("namespace") String namespace,
             final @PathParam("functionName") String functionName) throws IOException {
         return functions().getFunctionStatus(
-                tenant, namespace, functionName, uri.getRequestUri(), clientAppId(), clientAuthData());
+                tenant, namespace, functionName, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -178,7 +178,7 @@ public FunctionStatsImpl getFunctionStats(final @PathParam("tenant") String tena
                                               final @PathParam("namespace") String namespace,
                                               final @PathParam("functionName") String functionName) throws IOException {
         return functions().getFunctionStats(tenant, namespace, functionName,
-                uri.getRequestUri(), clientAppId(), clientAuthData());
+                uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -200,7 +200,7 @@ public FunctionInstanceStatsDataImpl getFunctionInstanceStats(
             final @PathParam("functionName") String functionName,
             final @PathParam("instanceId") String instanceId) throws IOException {
         return functions().getFunctionsInstanceStats(
-                tenant, namespace, functionName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+                tenant, namespace, functionName, instanceId, uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -213,7 +213,7 @@ public String triggerFunction(final @PathParam("tenant") String tenant,
                                   final @FormDataParam("dataStream") InputStream uploadedInputStream,
                                   final @FormDataParam("topic") String topic) {
         return functions().triggerFunction(tenant, namespace, functionName, input,
-                uploadedInputStream, topic, clientAppId(), clientAuthData());
+                uploadedInputStream, topic, authParams());
     }
 
     @POST
@@ -231,7 +231,7 @@ public void restartFunction(final @PathParam("tenant") String tenant,
                                 final @PathParam("functionName") String functionName,
                                 final @PathParam("instanceId") String instanceId) {
         functions().restartFunctionInstance(tenant, namespace, functionName, instanceId,
-                this.uri.getRequestUri(), clientAppId(), clientAuthData());
+                this.uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -246,7 +246,7 @@ public void restartFunction(final @PathParam("tenant") String tenant,
     public void restartFunction(final @PathParam("tenant") String tenant,
                                 final @PathParam("namespace") String namespace,
                                 final @PathParam("functionName") String functionName) {
-        functions().restartFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().restartFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -263,7 +263,7 @@ public void stopFunction(final @PathParam("tenant") String tenant,
                              final @PathParam("functionName") String functionName,
                              final @PathParam("instanceId") String instanceId) {
         functions().stopFunctionInstance(tenant, namespace, functionName, instanceId,
-                this.uri.getRequestUri(), clientAppId(), clientAuthData());
+                this.uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -278,7 +278,7 @@ public void stopFunction(final @PathParam("tenant") String tenant,
     public void stopFunction(final @PathParam("tenant") String tenant,
                              final @PathParam("namespace") String namespace,
                              final @PathParam("functionName") String functionName) {
-        functions().stopFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().stopFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -295,7 +295,7 @@ public void startFunction(final @PathParam("tenant") String tenant,
                               final @PathParam("functionName") String functionName,
                               final @PathParam("instanceId") String instanceId) {
         functions().startFunctionInstance(tenant, namespace, functionName, instanceId,
-                this.uri.getRequestUri(), clientAppId(), clientAuthData());
+                this.uri.getRequestUri(), authParams());
     }
 
     @POST
@@ -310,7 +310,7 @@ public void startFunction(final @PathParam("tenant") String tenant,
     public void startFunction(final @PathParam("tenant") String tenant,
                               final @PathParam("namespace") String namespace,
                               final @PathParam("functionName") String functionName) {
-        functions().startFunctionInstances(tenant, namespace, functionName, clientAppId(), clientAuthData());
+        functions().startFunctionInstances(tenant, namespace, functionName, authParams());
     }
 
     @POST
@@ -318,13 +318,13 @@ public void startFunction(final @PathParam("tenant") String tenant,
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     public void uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
                                final @FormDataParam("path") String path) {
-        functions().uploadFunction(uploadedInputStream, path, clientAppId(), clientAuthData());
+        functions().uploadFunction(uploadedInputStream, path, authParams());
     }
 
     @GET
     @Path("/download")
     public StreamingOutput downloadFunction(final @QueryParam("path") String path) {
-        return functions().downloadFunction(path, clientAppId(), clientAuthData());
+        return functions().downloadFunction(path, authParams());
     }
 
     @GET
@@ -344,7 +344,7 @@ public StreamingOutput downloadFunction(
             final @QueryParam("transform-function") boolean transformFunction) {
 
         return functions()
-                .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), transformFunction);
+                .downloadFunction(tenant, namespace, functionName, authParams(), transformFunction);
     }
 
     @GET
@@ -368,7 +368,7 @@ public List getConnectorsList() throws IOException {
     })
     @Path("/builtins/reload")
     public void reloadBuiltinFunctions() throws IOException {
-        functions().reloadBuiltinFunctions(clientAppId(), clientAuthData());
+        functions().reloadBuiltinFunctions(authParams());
     }
 
     @GET
@@ -385,7 +385,7 @@ public void reloadBuiltinFunctions() throws IOException {
     @Path("/builtins")
     @Produces(MediaType.APPLICATION_JSON)
     public List getBuiltinFunctions() {
-        return functions().getBuiltinFunctions(clientAppId(), clientAuthData());
+        return functions().getBuiltinFunctions(authParams());
     }
 
     @GET
@@ -394,7 +394,7 @@ public FunctionState getFunctionState(final @PathParam("tenant") String tenant,
                                           final @PathParam("namespace") String namespace,
                                           final @PathParam("functionName") String functionName,
                                           final @PathParam("key") String key) throws IOException {
-        return functions().getFunctionState(tenant, namespace, functionName, key, clientAppId(), clientAuthData());
+        return functions().getFunctionState(tenant, namespace, functionName, key, authParams());
     }
 
     @POST
@@ -405,7 +405,7 @@ public void putFunctionState(final @PathParam("tenant") String tenant,
                                  final @PathParam("functionName") String functionName,
                                  final @PathParam("key") String key,
                                  final @FormDataParam("state") FunctionState stateJson) throws IOException {
-        functions().putFunctionState(tenant, namespace, functionName, key, stateJson, clientAppId(), clientAuthData());
+        functions().putFunctionState(tenant, namespace, functionName, key, stateJson, authParams());
     }
 
     @PUT
@@ -427,6 +427,6 @@ public void updateFunctionOnWorkerLeader(final @PathParam("tenant") String tenan
                                              final @FormDataParam("delete") boolean delete) {
 
         functions().updateFunctionOnWorkerLeader(tenant, namespace, functionName, uploadedInputStream,
-                delete, uri.getRequestUri(), clientAppId(), clientAuthData());
+                delete, uri.getRequestUri(), authParams());
     }
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java
index b110c8f72c677..8081f76cfd8ec 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SinksApiV3Resource.java
@@ -70,7 +70,7 @@ public void registerSink(final @PathParam("tenant") String tenant,
                              final @FormDataParam("sinkConfig") SinkConfig sinkConfig) {
 
         sinks().registerSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail,
-                functionPkgUrl, sinkConfig, clientAppId(), clientAuthData());
+                functionPkgUrl, sinkConfig, authParams());
     }
 
     @PUT
@@ -86,7 +86,7 @@ public void updateSink(final @PathParam("tenant") String tenant,
                            final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) {
 
         sinks().updateSink(tenant, namespace, sinkName, uploadedInputStream, fileDetail,
-                functionPkgUrl, sinkConfig, clientAppId(), clientAuthData(), updateOptions);
+                functionPkgUrl, sinkConfig, authParams(), updateOptions);
     }
 
     @DELETE
@@ -94,7 +94,7 @@ public void updateSink(final @PathParam("tenant") String tenant,
     public void deregisterSink(final @PathParam("tenant") String tenant,
                                final @PathParam("namespace") String namespace,
                                final @PathParam("sinkName") String sinkName) {
-        sinks().deregisterFunction(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().deregisterFunction(tenant, namespace, sinkName, authParams());
     }
 
     @GET
@@ -103,7 +103,7 @@ public SinkConfig getSinkInfo(final @PathParam("tenant") String tenant,
                                   final @PathParam("namespace") String namespace,
                                   final @PathParam("sinkName") String sinkName)
             throws IOException {
-        return sinks().getSinkInfo(tenant, namespace, sinkName);
+        return sinks().getSinkInfo(tenant, namespace, sinkName, authParams());
     }
 
     @GET
@@ -124,9 +124,8 @@ public SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkInstanceStatu
             final @PathParam("namespace") String namespace,
             final @PathParam("sinkName") String sinkName,
             final @PathParam("instanceId") String instanceId) throws IOException {
-        return sinks()
-                .getSinkInstanceStatus(tenant, namespace, sinkName, instanceId, uri.getRequestUri(), clientAppId(),
-                        clientAuthData());
+        return sinks().getSinkInstanceStatus(tenant, namespace, sinkName, instanceId, uri.getRequestUri(),
+                authParams());
     }
 
     @GET
@@ -145,14 +144,14 @@ public SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkInstanceStatu
     public SinkStatus getSinkStatus(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace,
                                     final @PathParam("sinkName") String sinkName) throws IOException {
-        return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), clientAppId(), clientAuthData());
+        return sinks().getSinkStatus(tenant, namespace, sinkName, uri.getRequestUri(), authParams());
     }
 
     @GET
     @Path("/{tenant}/{namespace}")
     public List listSink(final @PathParam("tenant") String tenant,
                                  final @PathParam("namespace") String namespace) {
-        return sinks().listFunctions(tenant, namespace, clientAppId(), clientAuthData());
+        return sinks().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -169,7 +168,7 @@ public void restartSink(final @PathParam("tenant") String tenant,
                             final @PathParam("sinkName") String sinkName,
                             final @PathParam("instanceId") String instanceId) {
         sinks().restartFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(),
-                clientAppId(), clientAuthData());
+                authParams());
     }
 
     @POST
@@ -182,7 +181,7 @@ public void restartSink(final @PathParam("tenant") String tenant,
     public void restartSink(final @PathParam("tenant") String tenant,
                             final @PathParam("namespace") String namespace,
                             final @PathParam("sinkName") String sinkName) {
-        sinks().restartFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().restartFunctionInstances(tenant, namespace, sinkName, authParams());
     }
 
     @POST
@@ -196,8 +195,8 @@ public void stopSink(final @PathParam("tenant") String tenant,
                          final @PathParam("namespace") String namespace,
                          final @PathParam("sinkName") String sinkName,
                          final @PathParam("instanceId") String instanceId) {
-        sinks().stopFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), clientAppId(),
-                clientAuthData());
+        sinks().stopFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(),
+                authParams());
     }
 
     @POST
@@ -210,7 +209,7 @@ public void stopSink(final @PathParam("tenant") String tenant,
     public void stopSink(final @PathParam("tenant") String tenant,
                          final @PathParam("namespace") String namespace,
                          final @PathParam("sinkName") String sinkName) {
-        sinks().stopFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().stopFunctionInstances(tenant, namespace, sinkName, authParams());
     }
 
     @POST
@@ -224,8 +223,8 @@ public void startSink(final @PathParam("tenant") String tenant,
                           final @PathParam("namespace") String namespace,
                           final @PathParam("sinkName") String sinkName,
                           final @PathParam("instanceId") String instanceId) {
-        sinks().startFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(), clientAppId(),
-                clientAuthData());
+        sinks().startFunctionInstance(tenant, namespace, sinkName, instanceId, this.uri.getRequestUri(),
+                authParams());
     }
 
     @POST
@@ -238,7 +237,7 @@ public void startSink(final @PathParam("tenant") String tenant,
     public void startSink(final @PathParam("tenant") String tenant,
                           final @PathParam("namespace") String namespace,
                           final @PathParam("sinkName") String sinkName) {
-        sinks().startFunctionInstances(tenant, namespace, sinkName, clientAppId(), clientAuthData());
+        sinks().startFunctionInstances(tenant, namespace, sinkName, authParams());
     }
 
     @GET
@@ -278,6 +277,6 @@ public List getSinkConfigDefinition(
     })
     @Path("/reloadBuiltInSinks")
     public void reloadSinks() {
-        sinks().reloadConnectors(clientAppId(), clientAuthData());
+        sinks().reloadConnectors(authParams());
     }
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java
index a669d4f089dd0..23df423045090 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/SourcesApiV3Resource.java
@@ -70,7 +70,7 @@ public void registerSource(final @PathParam("tenant") String tenant,
                                    final @FormDataParam("sourceConfig") SourceConfig sourceConfig) {
 
         sources().registerSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail,
-                functionPkgUrl, sourceConfig, clientAppId(), clientAuthData());
+                functionPkgUrl, sourceConfig, authParams());
 
     }
 
@@ -87,7 +87,7 @@ public void updateSource(final @PathParam("tenant") String tenant,
                              final @FormDataParam("updateOptions") UpdateOptionsImpl updateOptions) {
 
         sources().updateSource(tenant, namespace, sourceName, uploadedInputStream, fileDetail,
-                functionPkgUrl, sourceConfig, clientAppId(), clientAuthData(), updateOptions);
+                functionPkgUrl, sourceConfig, authParams(), updateOptions);
     }
 
 
@@ -96,7 +96,7 @@ public void updateSource(final @PathParam("tenant") String tenant,
     public void deregisterSource(final @PathParam("tenant") String tenant,
                                  final @PathParam("namespace") String namespace,
                                  final @PathParam("sourceName") String sourceName) {
-        sources().deregisterFunction(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().deregisterFunction(tenant, namespace, sourceName, authParams());
     }
 
     @GET
@@ -106,7 +106,7 @@ public SourceConfig getSourceInfo(final @PathParam("tenant") String tenant,
                                       final @PathParam("namespace") String namespace,
                                       final @PathParam("sourceName") String sourceName)
             throws IOException {
-        return sources().getSourceInfo(tenant, namespace, sourceName);
+        return sources().getSourceInfo(tenant, namespace, sourceName, authParams());
     }
 
     @GET
@@ -127,8 +127,8 @@ public SourceStatus.SourceInstanceStatus.SourceInstanceStatusData getSourceInsta
             final @PathParam("namespace") String namespace,
             final @PathParam("sourceName") String sourceName,
             final @PathParam("instanceId") String instanceId) throws IOException {
-        return sources().getSourceInstanceStatus(
-            tenant, namespace, sourceName, instanceId, uri.getRequestUri(), clientAppId(), clientAuthData());
+        return sources().getSourceInstanceStatus(tenant, namespace, sourceName, instanceId, uri.getRequestUri(),
+                authParams());
     }
 
     @GET
@@ -148,7 +148,7 @@ public SourceStatus getSourceStatus(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace,
                                     final @PathParam("sourceName") String sourceName) throws IOException {
         return sources()
-                .getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), clientAppId(), clientAuthData());
+                .getSourceStatus(tenant, namespace, sourceName, uri.getRequestUri(), authParams());
     }
 
     @GET
@@ -156,7 +156,7 @@ public SourceStatus getSourceStatus(final @PathParam("tenant") String tenant,
     @Path("/{tenant}/{namespace}")
     public List listSources(final @PathParam("tenant") String tenant,
                                     final @PathParam("namespace") String namespace) {
-        return sources().listFunctions(tenant, namespace, clientAppId(), clientAuthData());
+        return sources().listFunctions(tenant, namespace, authParams());
     }
 
     @POST
@@ -173,7 +173,7 @@ public void restartSource(final @PathParam("tenant") String tenant,
                               final @PathParam("sourceName") String sourceName,
                               final @PathParam("instanceId") String instanceId) {
         sources().restartFunctionInstance(tenant, namespace, sourceName, instanceId, this.uri.getRequestUri(),
-                clientAppId(), clientAuthData());
+                authParams());
     }
 
     @POST
@@ -186,7 +186,7 @@ public void restartSource(final @PathParam("tenant") String tenant,
     public void restartSource(final @PathParam("tenant") String tenant,
                               final @PathParam("namespace") String namespace,
                               final @PathParam("sourceName") String sourceName) {
-        sources().restartFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().restartFunctionInstances(tenant, namespace, sourceName, authParams());
     }
 
     @POST
@@ -201,7 +201,7 @@ public void stopSource(final @PathParam("tenant") String tenant,
                            final @PathParam("sourceName") String sourceName,
                            final @PathParam("instanceId") String instanceId) {
         sources().stopFunctionInstance(tenant, namespace, sourceName, instanceId, this.uri.getRequestUri(),
-                clientAppId(), clientAuthData());
+                authParams());
     }
 
     @POST
@@ -214,7 +214,7 @@ public void stopSource(final @PathParam("tenant") String tenant,
     public void stopSource(final @PathParam("tenant") String tenant,
                            final @PathParam("namespace") String namespace,
                            final @PathParam("sourceName") String sourceName) {
-        sources().stopFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().stopFunctionInstances(tenant, namespace, sourceName, authParams());
     }
 
     @POST
@@ -229,7 +229,7 @@ public void startSource(final @PathParam("tenant") String tenant,
                             final @PathParam("sourceName") String sourceName,
                             final @PathParam("instanceId") String instanceId) {
         sources().startFunctionInstance(tenant, namespace, sourceName, instanceId, this.uri.getRequestUri(),
-                clientAppId(), clientAuthData());
+                authParams());
     }
 
     @POST
@@ -242,7 +242,7 @@ public void startSource(final @PathParam("tenant") String tenant,
     public void startSource(final @PathParam("tenant") String tenant,
                             final @PathParam("namespace") String namespace,
                             final @PathParam("sourceName") String sourceName) {
-        sources().startFunctionInstances(tenant, namespace, sourceName, clientAppId(), clientAuthData());
+        sources().startFunctionInstances(tenant, namespace, sourceName, authParams());
     }
 
     @GET
@@ -293,6 +293,6 @@ public List getSourceConfigDefinition(
     })
     @Path("/reloadBuiltInSources")
     public void reloadSources() {
-        sources().reloadConnectors(clientAppId(), clientAuthData());
+        sources().reloadConnectors(authParams());
     }
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java
index fc01ec55b6c4c..770cced1731da 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java
@@ -22,8 +22,7 @@
 import java.net.URI;
 import java.util.List;
 import javax.ws.rs.core.StreamingOutput;
-import org.apache.pulsar.broker.authentication.AuthenticationDataHttps;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.FunctionState;
 import org.apache.pulsar.common.io.ConnectorDefinition;
@@ -40,167 +39,55 @@ public interface Component {
 
     W worker();
 
-    void deregisterFunction(String tenant,
-                            String namespace,
-                            String componentName,
-                            String clientRole,
-                            AuthenticationDataSource clientAuthenticationDataHttps);
-
-    @Deprecated
-    default void deregisterFunction(String tenant,
-                            String namespace,
-                            String componentName,
-                            String clientRole,
-                            AuthenticationDataHttps clientAuthenticationDataHttps) {
-        deregisterFunction(
-                tenant,
-                namespace,
-                componentName,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps);
-    }
-
-    FunctionConfig getFunctionInfo(String tenant,
-                                   String namespace,
-                                   String componentName,
-                                   String clientRole,
-                                   AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void stopFunctionInstance(String tenant,
-                              String namespace,
-                              String componentName,
-                              String instanceId,
-                              URI uri,
-                              String clientRole,
-                              AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void startFunctionInstance(String tenant,
-                               String namespace,
-                               String componentName,
-                               String instanceId,
-                               URI uri,
-                               String clientRole,
-                               AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void restartFunctionInstance(String tenant,
-                                 String namespace,
-                                 String componentName,
-                                 String instanceId,
-                                 URI uri,
-                                 String clientRole,
-                                 AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void startFunctionInstances(String tenant,
-                                String namespace,
-                                String componentName,
-                                String clientRole,
-                                AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void stopFunctionInstances(String tenant,
-                               String namespace,
-                               String componentName,
-                               String clientRole,
-                               AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void restartFunctionInstances(String tenant,
-                                  String namespace,
-                                  String componentName,
-                                  String clientRole,
-                                  AuthenticationDataSource clientAuthenticationDataHttps);
-
-    FunctionStatsImpl getFunctionStats(String tenant,
-                                       String namespace,
-                                       String componentName,
-                                       URI uri,
-                                       String clientRole,
-                                       AuthenticationDataSource clientAuthenticationDataHttps);
-
-    FunctionInstanceStatsDataImpl getFunctionsInstanceStats(String tenant,
-                                                            String namespace,
-                                                            String componentName,
-                                                            String instanceId,
-                                                            URI uri,
-                                                            String clientRole,
-                                                            AuthenticationDataSource clientAuthenticationDataHttps);
-
-    String triggerFunction(String tenant,
-                           String namespace,
-                           String functionName,
-                           String input,
-                           InputStream uploadedInputStream,
-                           String topic,
-                           String clientRole,
-                           AuthenticationDataSource clientAuthenticationDataHttps);
-
-    List listFunctions(String tenant,
-                               String namespace,
-                               String clientRole,
-                               AuthenticationDataSource clientAuthenticationDataHttps);
-
-    FunctionState getFunctionState(String tenant,
-                                   String namespace,
-                                   String functionName,
-                                   String key,
-                                   String clientRole,
-                                   AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void putFunctionState(String tenant,
-                          String namespace,
-                          String functionName,
-                          String key,
-                          FunctionState state,
-                          String clientRole,
-                          AuthenticationDataSource clientAuthenticationDataHttps);
-
-    void uploadFunction(InputStream uploadedInputStream,
-                        String path,
-                        String clientRole,
-                        AuthenticationDataSource clientAuthenticationDataHttps);
-
-    StreamingOutput downloadFunction(String path,
-                                     String clientRole,
-                                     AuthenticationDataSource clientAuthenticationDataHttps);
-
-    @Deprecated
-    default StreamingOutput downloadFunction(String path,
-                                     String clientRole,
-                                     AuthenticationDataHttps clientAuthenticationDataHttps) {
-        return downloadFunction(path, clientRole, (AuthenticationDataSource) clientAuthenticationDataHttps);
-    }
-
-    StreamingOutput downloadFunction(String tenant,
-                                     String namespace,
-                                     String componentName,
-                                     String clientRole,
-                                     AuthenticationDataSource clientAuthenticationDataHttps,
-                                     boolean transformFunction);
-
-    @Deprecated
-    default StreamingOutput downloadFunction(String tenant,
-                                     String namespace,
-                                     String componentName,
-                                     String clientRole,
-                                     AuthenticationDataSource clientAuthenticationDataHttps) {
-        return downloadFunction(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps, false);
-    }
-
-    @Deprecated
-    default StreamingOutput downloadFunction(String tenant,
-                                     String namespace,
-                                     String componentName,
-                                     String clientRole,
-                                     AuthenticationDataHttps clientAuthenticationDataHttps) {
-        return downloadFunction(
-                tenant,
-                namespace,
-                componentName,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps,
-                false);
-    }
+    void deregisterFunction(String tenant, String namespace, String componentName, AuthenticationParameters authParams);
 
-    List getListOfConnectors();
+    FunctionConfig getFunctionInfo(String tenant, String namespace, String componentName,
+                                   AuthenticationParameters authParams);
+
+    void stopFunctionInstance(String tenant, String namespace, String componentName, String instanceId, URI uri,
+                              AuthenticationParameters authParams);
+
+    void startFunctionInstance(String tenant, String namespace, String componentName, String instanceId, URI uri,
+                               AuthenticationParameters authParams);
+
+    void restartFunctionInstance(String tenant, String namespace, String componentName, String instanceId, URI uri,
+                                 AuthenticationParameters authParams);
+
+    void startFunctionInstances(String tenant, String namespace, String componentName,
+                                AuthenticationParameters authParams);
+
+    void stopFunctionInstances(String tenant, String namespace, String componentName,
+                               AuthenticationParameters authParams);
+
+    void restartFunctionInstances(String tenant, String namespace, String componentName,
+                                  AuthenticationParameters authParams);
+
+    FunctionStatsImpl getFunctionStats(String tenant, String namespace, String componentName, URI uri,
+                                       AuthenticationParameters authParams);
+
+    FunctionInstanceStatsDataImpl getFunctionsInstanceStats(String tenant, String namespace, String componentName,
+                                                            String instanceId, URI uri,
+                                                            AuthenticationParameters authParams);
 
+    String triggerFunction(String tenant, String namespace, String functionName, String input,
+                           InputStream uploadedInputStream, String topic, AuthenticationParameters authParams);
+
+    List listFunctions(String tenant, String namespace, AuthenticationParameters authParams);
+
+    FunctionState getFunctionState(String tenant, String namespace, String functionName, String key,
+                                   AuthenticationParameters authParams);
+
+    void putFunctionState(String tenant, String namespace, String functionName, String key, FunctionState state,
+                          AuthenticationParameters authParams);
+
+    void uploadFunction(InputStream uploadedInputStream, String path, AuthenticationParameters authParams);
+
+    StreamingOutput downloadFunction(String path, AuthenticationParameters authParams);
+
+    StreamingOutput downloadFunction(String tenant, String namespace, String componentName,
+                                     AuthenticationParameters authParams, boolean transformFunction);
+
+    List getListOfConnectors();
 
-    void reloadConnectors(String clientRole, AuthenticationDataSource clientAuthenticationDataHttps);
+    void reloadConnectors(AuthenticationParameters authParams);
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java
index 954ab1d26182f..28bc73fd0a831 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Functions.java
@@ -22,8 +22,7 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.util.List;
-import org.apache.pulsar.broker.authentication.AuthenticationDataHttps;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.FunctionDefinition;
 import org.apache.pulsar.common.functions.UpdateOptionsImpl;
@@ -46,8 +45,7 @@ public interface Functions extends Component {
      * @param fileDetail A form-data content disposition header
      * @param functionPkgUrl URL path of the Pulsar Function package
      * @param functionConfig Configuration of Pulsar Function
-     * @param clientRole Client role for running the pulsar function
-     * @param clientAuthenticationDataHttps Authentication status of the http client
+     * @param authParams the authentication parameters associated with the request
      */
     void registerFunction(String tenant,
                           String namespace,
@@ -56,35 +54,7 @@ void registerFunction(String tenant,
                           FormDataContentDisposition fileDetail,
                           String functionPkgUrl,
                           FunctionConfig functionConfig,
-                          String clientRole,
-                          AuthenticationDataSource clientAuthenticationDataHttps);
-
-    /**
-     * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status,
-     * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts
-     * the signature of the AuthenticationDataSource.
-     */
-    @Deprecated
-    default void registerFunction(String tenant,
-                          String namespace,
-                          String functionName,
-                          InputStream uploadedInputStream,
-                          FormDataContentDisposition fileDetail,
-                          String functionPkgUrl,
-                          FunctionConfig functionConfig,
-                          String clientRole,
-                          AuthenticationDataHttps clientAuthenticationDataHttps) {
-        registerFunction(
-                tenant,
-                namespace,
-                functionName,
-                uploadedInputStream,
-                fileDetail,
-                functionPkgUrl,
-                functionConfig,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps);
-    }
+                          AuthenticationParameters authParams);
 
     /**
      * Update a function.
@@ -95,8 +65,7 @@ default void registerFunction(String tenant,
      * @param fileDetail A form-data content disposition header
      * @param functionPkgUrl URL path of the Pulsar Function package
      * @param functionConfig Configuration of Pulsar Function
-     * @param clientRole Client role for running the Pulsar Function
-     * @param clientAuthenticationDataHttps Authentication status of the http client
+     * @param authParams the authentication parameters associated with the request
      * @param updateOptions Options while updating the function
      */
     void updateFunction(String tenant,
@@ -106,66 +75,31 @@ void updateFunction(String tenant,
                         FormDataContentDisposition fileDetail,
                         String functionPkgUrl,
                         FunctionConfig functionConfig,
-                        String clientRole,
-                        AuthenticationDataSource clientAuthenticationDataHttps,
+                        AuthenticationParameters authParams,
                         UpdateOptionsImpl updateOptions);
 
-    /**
-     * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status,
-     * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts
-     * the signature of the AuthenticationDataSource.
-     */
-    @Deprecated
-    default void updateFunction(String tenant,
-                        String namespace,
-                        String functionName,
-                        InputStream uploadedInputStream,
-                        FormDataContentDisposition fileDetail,
-                        String functionPkgUrl,
-                        FunctionConfig functionConfig,
-                        String clientRole,
-                        AuthenticationDataHttps clientAuthenticationDataHttps,
-                        UpdateOptionsImpl updateOptions) {
-        updateFunction(
-                tenant,
-                namespace,
-                functionName,
-                uploadedInputStream,
-                fileDetail,
-                functionPkgUrl,
-                functionConfig,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps,
-                updateOptions);
-    }
-
     void updateFunctionOnWorkerLeader(String tenant,
                                       String namespace,
                                       String functionName,
                                       InputStream uploadedInputStream,
                                       boolean delete,
                                       URI uri,
-                                      String clientRole,
-                                      AuthenticationDataSource clientAuthenticationDataHttps);
+                                      AuthenticationParameters authParams);
 
     FunctionStatus getFunctionStatus(String tenant,
                                      String namespace,
                                      String componentName,
                                      URI uri,
-                                     String clientRole,
-                                     AuthenticationDataSource clientAuthenticationDataHttps);
+                                     AuthenticationParameters authParams);
 
     FunctionInstanceStatusData getFunctionInstanceStatus(String tenant,
                                                          String namespace,
                                                          String componentName,
                                                          String instanceId,
                                                          URI uri,
-                                                         String clientRole,
-                                                         AuthenticationDataSource clientAuthenticationDataHttps);
+                                                         AuthenticationParameters authParams);
 
-    void reloadBuiltinFunctions(String clientRole, AuthenticationDataSource clientAuthenticationDataHttps)
-        throws IOException;
+    void reloadBuiltinFunctions(AuthenticationParameters authParams) throws IOException;
 
-    List getBuiltinFunctions(String clientRole,
-                                                 AuthenticationDataSource clientAuthenticationDataHttps);
+    List getBuiltinFunctions(AuthenticationParameters authParams);
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java
index 6a4be7cdf6f0e..d78d241b3e6a0 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/FunctionsV2.java
@@ -23,6 +23,7 @@
 import java.net.URI;
 import java.util.List;
 import javax.ws.rs.core.Response;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.common.io.ConnectorDefinition;
 import org.apache.pulsar.functions.worker.WorkerService;
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -35,20 +36,20 @@ public interface FunctionsV2 {
     Response getFunctionInfo(String tenant,
                              String namespace,
                              String functionName,
-                             String clientRole) throws IOException;
+                             AuthenticationParameters authParams) throws IOException;
 
     Response getFunctionInstanceStatus(String tenant,
-                                       String namespace,
-                                       String functionName,
-                                       String instanceId,
-                                       URI uri,
-                                       String clientRole) throws IOException;
+                                               String namespace,
+                                               String functionName,
+                                               String instanceId,
+                                               URI uri,
+                                               AuthenticationParameters authParams) throws IOException;
 
     Response getFunctionStatusV2(String tenant,
                                  String namespace,
                                  String functionName,
                                  URI requestUri,
-                                 String clientRole) throws IOException;
+                                 AuthenticationParameters authParams) throws IOException;
 
     Response registerFunction(String tenant,
                               String namespace,
@@ -57,7 +58,8 @@ Response registerFunction(String tenant,
                               FormDataContentDisposition fileDetail,
                               String functionPkgUrl,
                               String functionDetailsJson,
-                              String clientRole);
+                              AuthenticationParameters authParams);
+
 
     Response updateFunction(String tenant,
                             String namespace,
@@ -66,14 +68,12 @@ Response updateFunction(String tenant,
                             FormDataContentDisposition fileDetail,
                             String functionPkgUrl,
                             String functionDetailsJson,
-                            String clientRole);
+                            AuthenticationParameters authParams);
 
-    Response deregisterFunction(String tenant,
-                                String namespace,
-                                String functionName,
-                                String clientAppId);
+    Response deregisterFunction(String tenant, String namespace, String functionName,
+                                AuthenticationParameters authParams);
 
-    Response listFunctions(String tenant, String namespace, String clientRole);
+    Response listFunctions(String tenant, String namespace, AuthenticationParameters authParams);
 
     Response triggerFunction(String tenant,
                              String namespace,
@@ -81,45 +81,44 @@ Response triggerFunction(String tenant,
                              String triggerValue,
                              InputStream triggerStream,
                              String topic,
-                             String clientRole);
+                             AuthenticationParameters authParams);
 
     Response getFunctionState(String tenant,
                               String namespace,
                               String functionName,
                               String key,
-                              String clientRole);
-
+                              AuthenticationParameters authParams);
 
     Response restartFunctionInstance(String tenant,
                                      String namespace,
                                      String functionName,
                                      String instanceId,
                                      URI uri,
-                                     String clientRole);
+                                     AuthenticationParameters authParams);
+
 
     Response restartFunctionInstances(String tenant,
                                       String namespace,
                                       String functionName,
-                                      String clientRole);
+                                      AuthenticationParameters authParams);
 
     Response stopFunctionInstance(String tenant,
                                   String namespace,
                                   String functionName,
                                   String instanceId,
                                   URI uri,
-                                  String clientRole);
+                                  AuthenticationParameters authParams);
 
     Response stopFunctionInstances(String tenant,
                                    String namespace,
                                    String functionName,
-                                   String clientRole);
+                                   AuthenticationParameters authParams);
 
     Response uploadFunction(InputStream uploadedInputStream,
                             String path,
-                            String clientRole);
-
-    Response downloadFunction(String path, String clientRole);
+                            AuthenticationParameters authParams);
 
+    Response downloadFunction(String path, AuthenticationParameters authParams);
 
     List getListOfConnectors();
 
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java
index 2c0dcb82e0dbc..c977f55df4bf3 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sinks.java
@@ -21,8 +21,7 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.util.List;
-import org.apache.pulsar.broker.authentication.AuthenticationDataHttps;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.common.functions.UpdateOptionsImpl;
 import org.apache.pulsar.common.io.ConfigFieldDefinition;
 import org.apache.pulsar.common.io.ConnectorDefinition;
@@ -46,8 +45,7 @@ public interface Sinks extends Component {
      * @param fileDetail A form-data content disposition header
      * @param sinkPkgUrl URL path of the Pulsar Sink package
      * @param sinkConfig Configuration of Pulsar Sink
-     * @param clientRole Client role for running the Pulsar Sink
-     * @param clientAuthenticationDataHttps Authentication status of the http client
+     * @param authParams the authentication parameters associated with the request
      */
     void registerSink(String tenant,
                       String namespace,
@@ -56,35 +54,7 @@ void registerSink(String tenant,
                       FormDataContentDisposition fileDetail,
                       String sinkPkgUrl,
                       SinkConfig sinkConfig,
-                      String clientRole,
-                      AuthenticationDataSource clientAuthenticationDataHttps);
-
-    /**
-     * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status,
-     * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts
-     * the signature of the AuthenticationDataSource.
-     */
-    @Deprecated
-    default void registerSink(String tenant,
-                      String namespace,
-                      String sinkName,
-                      InputStream uploadedInputStream,
-                      FormDataContentDisposition fileDetail,
-                      String sinkPkgUrl,
-                      SinkConfig sinkConfig,
-                      String clientRole,
-                      AuthenticationDataHttps clientAuthenticationDataHttps) {
-        registerSink(
-                tenant,
-                namespace,
-                sinkName,
-                uploadedInputStream,
-                fileDetail,
-                sinkPkgUrl,
-                sinkConfig,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps);
-    }
+                      AuthenticationParameters authParams);
 
     /**
      * Update a function.
@@ -95,8 +65,7 @@ default void registerSink(String tenant,
      * @param fileDetail A form-data content disposition header
      * @param sinkPkgUrl URL path of the Pulsar Sink package
      * @param sinkConfig Configuration of Pulsar Sink
-     * @param clientRole Client role for running the Pulsar Sink
-     * @param clientAuthenticationDataHttps Authentication status of the http client
+     * @param authParams the authentication parameters associated with the request
      * @param updateOptions Options while updating the sink
      */
     void updateSink(String tenant,
@@ -106,57 +75,26 @@ void updateSink(String tenant,
                     FormDataContentDisposition fileDetail,
                     String sinkPkgUrl,
                     SinkConfig sinkConfig,
-                    String clientRole,
-                    AuthenticationDataSource clientAuthenticationDataHttps,
+                    AuthenticationParameters authParams,
                     UpdateOptionsImpl updateOptions);
 
-    /**
-     * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status,
-     * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts
-     * the signature of the AuthenticationDataSource.
-     */
-    @Deprecated
-    default void updateSink(String tenant,
-                    String namespace,
-                    String sinkName,
-                    InputStream uploadedInputStream,
-                    FormDataContentDisposition fileDetail,
-                    String sinkPkgUrl,
-                    SinkConfig sinkConfig,
-                    String clientRole,
-                    AuthenticationDataHttps clientAuthenticationDataHttps,
-                    UpdateOptionsImpl updateOptions) {
-        updateSink(
-                tenant,
-                namespace,
-                sinkName,
-                uploadedInputStream,
-                fileDetail,
-                sinkPkgUrl,
-                sinkConfig,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps,
-                updateOptions);
-    }
-
     SinkInstanceStatusData getSinkInstanceStatus(String tenant,
                                                  String namespace,
                                                  String sinkName,
                                                  String instanceId,
                                                  URI uri,
-                                                 String clientRole,
-                                                 AuthenticationDataSource clientAuthenticationDataHttps);
+                                                 AuthenticationParameters authParams);
 
     SinkStatus getSinkStatus(String tenant,
                              String namespace,
                              String componentName,
                              URI uri,
-                             String clientRole,
-                             AuthenticationDataSource clientAuthenticationDataHttps);
+                             AuthenticationParameters authParams);
 
     SinkConfig getSinkInfo(String tenant,
                            String namespace,
-                           String componentName);
+                           String componentName,
+                           AuthenticationParameters authParams);
 
     List getSinkList();
 
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java
index 781a25e92cc87..7fe1e56bf9a13 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Sources.java
@@ -21,8 +21,7 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.util.List;
-import org.apache.pulsar.broker.authentication.AuthenticationDataHttps;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.common.functions.UpdateOptionsImpl;
 import org.apache.pulsar.common.io.ConfigFieldDefinition;
 import org.apache.pulsar.common.io.ConnectorDefinition;
@@ -46,8 +45,7 @@ public interface Sources extends Component {
      * @param fileDetail A form-data content disposition header
      * @param sourcePkgUrl URL path of the Pulsar Source package
      * @param sourceConfig Configuration of Pulsar Source
-     * @param clientRole Client role for running the Pulsar Source
-     * @param clientAuthenticationDataHttps Authentication status of the http client
+     * @param authParams the authentication parameters associated with the request
      */
     void registerSource(String tenant,
                         String namespace,
@@ -56,35 +54,7 @@ void registerSource(String tenant,
                         FormDataContentDisposition fileDetail,
                         String sourcePkgUrl,
                         SourceConfig sourceConfig,
-                        String clientRole,
-                        AuthenticationDataSource clientAuthenticationDataHttps);
-
-    /**
-     * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status,
-     * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts
-     * the signature of the AuthenticationDataSource.
-     */
-    @Deprecated
-    default void registerSource(String tenant,
-                        String namespace,
-                        String sourceName,
-                        InputStream uploadedInputStream,
-                        FormDataContentDisposition fileDetail,
-                        String sourcePkgUrl,
-                        SourceConfig sourceConfig,
-                        String clientRole,
-                        AuthenticationDataHttps clientAuthenticationDataHttps) {
-        registerSource(
-                tenant,
-                namespace,
-                sourceName,
-                uploadedInputStream,
-                fileDetail,
-                sourcePkgUrl,
-                sourceConfig,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps);
-    }
+                        AuthenticationParameters authParams);
 
     /**
      * Update a function.
@@ -95,8 +65,7 @@ default void registerSource(String tenant,
      * @param fileDetail A form-data content disposition header
      * @param sourcePkgUrl URL path of the Pulsar Source package
      * @param sourceConfig Configuration of Pulsar Source
-     * @param clientRole Client role for running the Pulsar Source
-     * @param clientAuthenticationDataHttps Authentication status of the http client
+     * @param authParams the authentication parameters associated with the request
      * @param updateOptions Options while updating the source
      */
     void updateSource(String tenant,
@@ -106,59 +75,26 @@ void updateSource(String tenant,
                       FormDataContentDisposition fileDetail,
                       String sourcePkgUrl,
                       SourceConfig sourceConfig,
-                      String clientRole,
-                      AuthenticationDataSource clientAuthenticationDataHttps,
+                      AuthenticationParameters authParams,
                       UpdateOptionsImpl updateOptions);
 
-    /**
-     * This method uses an incorrect signature 'AuthenticationDataHttps' that prevents the extension of auth status,
-     * so it is marked as deprecated and kept here only for backward compatibility. Please use the method that accepts
-     * the signature of the AuthenticationDataSource.
-     */
-    @Deprecated
-    default void updateSource(String tenant,
-                      String namespace,
-                      String sourceName,
-                      InputStream uploadedInputStream,
-                      FormDataContentDisposition fileDetail,
-                      String sourcePkgUrl,
-                      SourceConfig sourceConfig,
-                      String clientRole,
-                      AuthenticationDataHttps clientAuthenticationDataHttps,
-                      UpdateOptionsImpl updateOptions) {
-        updateSource(
-                tenant,
-                namespace,
-                sourceName,
-                uploadedInputStream,
-                fileDetail,
-                sourcePkgUrl,
-                sourceConfig,
-                clientRole,
-                (AuthenticationDataSource) clientAuthenticationDataHttps,
-                updateOptions);
-    }
-
-
     SourceStatus getSourceStatus(String tenant,
                                  String namespace,
                                  String componentName,
                                  URI uri,
-                                 String clientRole,
-                                 AuthenticationDataSource clientAuthenticationDataHttps);
-
+                                 AuthenticationParameters authParams);
 
     SourceInstanceStatusData getSourceInstanceStatus(String tenant,
                                                      String namespace,
                                                      String sourceName,
                                                      String instanceId,
                                                      URI uri,
-                                                     String clientRole,
-                                                     AuthenticationDataSource clientAuthenticationDataHttps);
+                                                     AuthenticationParameters authParams);
 
     SourceConfig getSourceInfo(String tenant,
                                String namespace,
-                               String componentName);
+                               String componentName,
+                               AuthenticationParameters authParams);
 
     List getSourceList();
 
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java
index a13157ea6476c..420fdcefe8b57 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Workers.java
@@ -23,6 +23,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.LongRunningProcessStatus;
 import org.apache.pulsar.common.functions.WorkerInfo;
 import org.apache.pulsar.common.io.ConnectorDefinition;
@@ -35,25 +36,25 @@
  */
 public interface Workers {
 
-    List getCluster(String clientRole);
+    List getCluster(AuthenticationParameters authParams);
 
-    WorkerInfo getClusterLeader(String clientRole);
+    WorkerInfo getClusterLeader(AuthenticationParameters authParams);
 
-    Map> getAssignments(String clientRole);
+    Map> getAssignments(AuthenticationParameters authParams);
 
-    List getWorkerMetrics(String clientRole);
+    List getWorkerMetrics(AuthenticationParameters authParams);
 
-    List getFunctionsMetrics(String clientRole) throws IOException;
+    List getFunctionsMetrics(AuthenticationParameters authParams) throws IOException;
 
-    List getListOfConnectors(String clientRole);
+    List getListOfConnectors(AuthenticationParameters authParams);
 
-    void rebalance(URI uri, String clientRole);
+    void rebalance(URI uri, AuthenticationParameters authParams);
 
-    void drain(URI uri, String workerId, String clientRole, boolean leaderUri);
+    void drain(URI uri, String workerId, AuthenticationParameters authParams, boolean leaderUri);
 
-    LongRunningProcessStatus getDrainStatus(URI uri, String workerId, String clientRole,
+    LongRunningProcessStatus getDrainStatus(URI uri, String workerId, AuthenticationParameters authParams,
                                             boolean leaderUri);
 
-    Boolean isLeaderReady(String clientRole);
+    boolean isLeaderReady(AuthenticationParameters authParams);
 
 }
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java
index c5e6b4aaded74..0f5fca4a8a5a3 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java
@@ -33,17 +33,20 @@
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.HashSet;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import org.apache.distributedlog.DistributedLogConfiguration;
+import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerAccessMode;
 import org.apache.pulsar.client.api.ProducerBuilder;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
 import org.testng.annotations.Test;
 
 public class WorkerUtilsTest {
@@ -118,4 +121,15 @@ public void testDLogConfiguration() throws URISyntaxException, IOException {
         assertEquals(dlogConf.getString("bkc.testKey"), "fakeValue",
                 "The bookkeeper client config mapping should apply.");
     }
+
+    @Test
+    public void testProxyRolesInWorkerConfigMapToServiceConfiguration() throws Exception {
+        URL yamlUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
+        WorkerConfig wc = WorkerConfig.load(yamlUrl.toURI().getPath());
+        ServiceConfiguration conf = PulsarConfigurationLoader.convertFrom(wc);
+        HashSet proxyRoles = new HashSet<>();
+        proxyRoles.add("proxyA");
+        proxyRoles.add("proxyB");
+        assertEquals(conf.getProxyRoles(), proxyRoles);
+    }
 }
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java
index 998d85683e204..cfe087c78406a 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.functions.worker.rest.api;
 
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.anyInt;
 import static org.mockito.Mockito.doReturn;
@@ -28,27 +27,39 @@
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
 import java.io.InputStream;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
 import org.apache.distributedlog.api.namespace.Namespace;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.authorization.AuthorizationService;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.broker.resources.TenantResources;
 import org.apache.pulsar.client.admin.Namespaces;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.admin.Tenants;
 import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
 import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.policies.data.AuthAction;
 import org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl;
 import org.apache.pulsar.common.policies.data.FunctionStatsImpl;
+import org.apache.pulsar.common.policies.data.Policies;
 import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.apache.pulsar.common.util.RestException;
 import org.apache.pulsar.functions.api.Context;
 import org.apache.pulsar.functions.instance.InstanceConfig;
 import org.apache.pulsar.functions.instance.JavaInstanceRunnable;
@@ -94,6 +105,7 @@ public String process(String input, Context context) {
     private static final int parallelism = 1;
     private static final String workerId = "worker-0";
     private static final String superUser = "superUser";
+    private static final String proxyUser = "proxyUser";
 
     private PulsarWorkerService mockedWorkerService;
     private PulsarAdmin mockedPulsarAdmin;
@@ -196,7 +208,7 @@ public void cleanup() {
 
     @Test
     public void testStatusEmpty() {
-        assertNotNull(this.resource.getFunctionInstanceStatus(tenant, namespace, function, "0", null, null, null));
+        assertNotNull(this.resource.getFunctionInstanceStatus(tenant, namespace, function, "0", null, null));
     }
 
     @Test
@@ -231,83 +243,80 @@ public void testMetricsEmpty() throws PulsarClientException {
         assertNotNull(functionStats.calculateOverall());
     }
 
+    // Suppress the deprecation warnings until we actually remove the deprecated method
+    @SuppressWarnings("deprecation")
     @Test
-    public void testIsAuthorizedRole() throws PulsarAdminException, InterruptedException, ExecutionException {
+    public void testIsAuthorizedRole() throws Exception {
 
-        TenantInfo tenantInfo = TenantInfo.builder().build();
         AuthenticationDataSource authenticationDataSource = mock(AuthenticationDataSource.class);
         FunctionsImpl functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService));
-        AuthorizationService authorizationService = mock(AuthorizationService.class);
-        doReturn(authorizationService).when(mockedWorkerService).getAuthorizationService();
         WorkerConfig workerConfig = new WorkerConfig();
         workerConfig.setAuthorizationEnabled(true);
-        workerConfig.setSuperUserRoles(Collections.singleton(superUser));
+        HashSet superUsers = new HashSet<>();
+        superUsers.add(superUser);
+        superUsers.add(proxyUser);
+        workerConfig.setSuperUserRoles(superUsers);
+        workerConfig.setProxyRoles(Collections.singleton(proxyUser));
+        // TODO remove mocking by relying on TestPulsarResources. Can't do now because this commit needs to be
+        //  cherry picked back.
+        PulsarResources pulsarResources = mock(PulsarResources.class);
+        TenantResources tenantResources = mock(TenantResources.class);
+        when(pulsarResources.getTenantResources()).thenReturn(tenantResources);
+        TenantInfo tenantInfo = TenantInfo.builder().adminRoles(Collections.singleton("tenant-admin")).build();
+        when(tenantResources.getTenantAsync("test-tenant"))
+                .thenReturn(CompletableFuture.completedFuture(Optional.of(tenantInfo)));
+        NamespaceResources namespaceResources = mock(NamespaceResources.class);
+        when(pulsarResources.getNamespaceResources()).thenReturn(namespaceResources);
+        Policies p = new Policies();
+        p.auth_policies.getNamespaceAuthentication().put("test-function-user", Set.of(AuthAction.functions));
+        when(namespaceResources.getPoliciesAsync(NamespaceName.get("test-tenant/test-ns")))
+                .thenReturn(CompletableFuture.completedFuture(Optional.of(p)));
+
+        AuthorizationService authorizationService = new AuthorizationService(
+                PulsarConfigurationLoader.convertFrom(workerConfig), pulsarResources);
         doReturn(workerConfig).when(mockedWorkerService).getWorkerConfig();
+        doReturn(authorizationService).when(mockedWorkerService).getAuthorizationService();
 
         // test super user
-        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", superUser, authenticationDataSource));
-
-        // test pulsar super user
-        final String pulsarSuperUser = "pulsarSuperUser";
-        when(authorizationService.isSuperUser(eq(pulsarSuperUser), any()))
-            .thenReturn(CompletableFuture.completedFuture(true));
-        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", pulsarSuperUser, authenticationDataSource));
-        assertTrue(functionImpl.isSuperUser(pulsarSuperUser, null));
+        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", superUser,
+                authenticationDataSource));
+        assertTrue(functionImpl.isSuperUser(superUser, null));
 
-        // test normal user
-        functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService));
-        doReturn(false).when(functionImpl).allowFunctionOps(any(), any(), any());
-        Tenants tenants = mock(Tenants.class);
-        when(tenants.getTenantInfo(any())).thenReturn(tenantInfo);
-        PulsarAdmin admin = mock(PulsarAdmin.class);
-        when(admin.tenants()).thenReturn(tenants);
-        when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin);
-        when(authorizationService.isTenantAdmin("test-tenant", "test-user", tenantInfo, authenticationDataSource))
-                .thenReturn(CompletableFuture.completedFuture(false));
-        when(authorizationService.isSuperUser(eq("test-user"), any()))
-                .thenReturn(CompletableFuture.completedFuture(false));
-        assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-user", authenticationDataSource));
+        // test normal user with no permissions
+        assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-non-admin-user",
+                authenticationDataSource));
 
         // if user is tenant admin
-        functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService));
-        doReturn(false).when(functionImpl).allowFunctionOps(any(), any(), any());
-        tenants = mock(Tenants.class);
-        tenantInfo = TenantInfo.builder().adminRoles(Collections.singleton("test-user")).build();
-        when(tenants.getTenantInfo(any())).thenReturn(tenantInfo);
-
-        admin = mock(PulsarAdmin.class);
-        when(admin.tenants()).thenReturn(tenants);
-        when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin);
-        when(authorizationService.isTenantAdmin("test-tenant", "test-user", tenantInfo, authenticationDataSource))
-                .thenReturn(CompletableFuture.completedFuture(true));
-        when(authorizationService.isSuperUser("test-user", authenticationDataSource))
-                .thenReturn(CompletableFuture.completedFuture(false));
-        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-user", authenticationDataSource));
+        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "tenant-admin",
+                authenticationDataSource));
 
         // test user allow function action
-        functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService));
-        doReturn(true).when(functionImpl).allowFunctionOps(any(), any(), any());
-        tenants = mock(Tenants.class);
-        tenantInfo = TenantInfo.builder().build();
-        when(tenants.getTenantInfo(any())).thenReturn(tenantInfo);
-
-        admin = mock(PulsarAdmin.class);
-        when(admin.tenants()).thenReturn(tenants);
-        when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin);
-        when(authorizationService.isTenantAdmin("test-tenant", "test-user", tenantInfo, authenticationDataSource))
-                .thenReturn(CompletableFuture.completedFuture(true));
-        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-user", authenticationDataSource));
+        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns", "test-function-user",
+                authenticationDataSource));
 
         // test role is null
-        functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService));
-        doReturn(true).when(functionImpl).allowFunctionOps(any(), any(), any());
-        tenants = mock(Tenants.class);
-        when(tenants.getTenantInfo(any())).thenReturn(TenantInfo.builder().build());
-
-        admin = mock(PulsarAdmin.class);
-        when(admin.tenants()).thenReturn(tenants);
-        when(this.mockedWorkerService.getBrokerAdmin()).thenReturn(admin);
-        assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns", null, authenticationDataSource));
+        assertThrows(RestException.class, () -> functionImpl.isAuthorizedRole("test-tenant",
+                "test-ns", null, authenticationDataSource));
+
+        // test proxy user with no original principal
+        assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns",
+                AuthenticationParameters.builder().clientRole(proxyUser).build()));
+
+        // test proxy user with tenant admin original principal
+        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns",
+                AuthenticationParameters.builder().clientRole(proxyUser).originalPrincipal("tenant-admin").build()));
+
+        // test proxy user with non admin user
+        assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns",
+                AuthenticationParameters.builder().clientRole(proxyUser).originalPrincipal("test-non-admin-user").build()));
+
+        // test proxy user with allow function action
+        assertTrue(functionImpl.isAuthorizedRole("test-tenant", "test-ns",
+                AuthenticationParameters.builder().clientRole(proxyUser).originalPrincipal("test-function-user").build()));
+
+        // test non-proxy user passing original principal
+        assertFalse(functionImpl.isAuthorizedRole("test-tenant", "test-ns",
+                AuthenticationParameters.builder().clientRole("nobody").originalPrincipal("test-non-admin-user").build()));
     }
 
     @Test
@@ -320,20 +329,19 @@ public void testIsSuperUser() throws PulsarAdminException {
         workerConfig.setAuthorizationEnabled(true);
         workerConfig.setSuperUserRoles(Collections.singleton(superUser));
         doReturn(workerConfig).when(mockedWorkerService).getWorkerConfig();
-        when(authorizationService.isSuperUser(anyString(), any()))
+        when(authorizationService.isSuperUser(any(AuthenticationParameters.class)))
             .thenAnswer((invocationOnMock) -> {
-                String role = invocationOnMock.getArgument(0, String.class);
+                String role = invocationOnMock.getArgument(0, AuthenticationParameters.class).getClientRole();
                 return CompletableFuture.completedFuture(superUser.equals(role));
             });
 
-        AuthenticationDataSource authenticationDataSource = mock(AuthenticationDataSource.class);
         assertTrue(functionImpl.isSuperUser(superUser, null));
 
         assertFalse(functionImpl.isSuperUser("normal-user", null));
         assertFalse(functionImpl.isSuperUser(null, null));
 
         // test super roles is null and it's not a pulsar super user
-        when(authorizationService.isSuperUser(superUser, null))
+        when(authorizationService.isSuperUser(AuthenticationParameters.builder().clientRole(superUser).build()))
                 .thenReturn(CompletableFuture.completedFuture(false));
         functionImpl = spy(new FunctionsImpl(() -> mockedWorkerService));
         workerConfig = new WorkerConfig();
@@ -342,10 +350,10 @@ public void testIsSuperUser() throws PulsarAdminException {
         assertFalse(functionImpl.isSuperUser(superUser, null));
 
         // test super role is null but the auth datasource contains superuser
-        when(authorizationService.isSuperUser(anyString(), any(AuthenticationDataSource.class)))
+        when(authorizationService.isSuperUser(any(AuthenticationParameters.class)))
             .thenAnswer((invocationOnMock -> {
-                AuthenticationDataSource authData = invocationOnMock.getArgument(1, AuthenticationDataSource.class);
-                String user = authData.getHttpHeader("mockedUser");
+                AuthenticationParameters authData = invocationOnMock.getArgument(0, AuthenticationParameters.class);
+                String user = authData.getClientAuthenticationDataSource().getHttpHeader("mockedUser");
                 return CompletableFuture.completedFuture(superUser.equals(user));
             }));
         AuthenticationDataSource authData = mock(AuthenticationDataSource.class);
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java
new file mode 100644
index 0000000000000..eb7228383552c
--- /dev/null
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/WorkerImplTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.worker.rest.api;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import java.util.HashSet;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
+import org.apache.pulsar.broker.authorization.AuthorizationService;
+import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
+import org.apache.pulsar.common.util.RestException;
+import org.apache.pulsar.functions.worker.LeaderService;
+import org.apache.pulsar.functions.worker.MetricsGenerator;
+import org.apache.pulsar.functions.worker.PulsarWorkerService;
+import org.apache.pulsar.functions.worker.WorkerConfig;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test of {@link WorkerImpl} to ensure that all methods properly fail if the {@link AuthenticationParameters}
+ * do not represent a superuser.
+ */
+public class WorkerImplTest {
+    WorkerImpl worker;
+
+    @DataProvider(name = "authParamsForNonSuperusers")
+    public Object[][] partitionedTopicProvider() {
+        return new Object[][] {
+                { AuthenticationParameters.builder().build() }, // all fields null should fail
+                { AuthenticationParameters.builder().clientRole("user").build() }, // Not superuser, no proxy
+                { AuthenticationParameters.builder()
+                        .clientRole("proxySuperuser").build() }, // Proxy role needs original principal
+                { AuthenticationParameters.builder()
+                        .clientRole("proxyNotSuperuser").build() }, // Proxy role needs original principal
+                { AuthenticationParameters.builder().clientRole("proxyNotSuperuser")
+                        .originalPrincipal("superuser").build() }, // Proxy is not superuser
+                { AuthenticationParameters.builder().clientRole("proxyNotSuperuser")
+                        .originalPrincipal("user").build() }, // Neither proxy nor original principal is superuser
+                { AuthenticationParameters.builder().clientRole("proxySuperuser")
+                        .originalPrincipal("user").build() }, // Original principal is not superuser
+                { AuthenticationParameters.builder().clientRole("user")
+                        .originalPrincipal("superuser").build() }, // User is not a proxyRole
+                { AuthenticationParameters.builder().clientRole("superuser2")
+                        .originalPrincipal("superuser").build() }, // Both are superusers, but client is not a proxyRole
+        };
+    }
+
+    @BeforeClass
+    void setup() throws Exception {
+        WorkerConfig config = new WorkerConfig();
+        config.setPulsarFunctionsCluster("testThrowIfNotSuperUserFailures");
+        config.setAuthorizationEnabled(true);
+        HashSet proxyRoles = new HashSet<>();
+        proxyRoles.add("proxySuperuser");
+        proxyRoles.add("proxyNotSuperuser");
+        HashSet superUserRoles = new HashSet<>();
+        superUserRoles.add("superuser");
+        superUserRoles.add("superuser2");
+        superUserRoles.add("proxySuperuser");
+        config.setSuperUserRoles(superUserRoles);
+        config.setProxyRoles(proxyRoles);
+        AuthorizationService authorizationService = new AuthorizationService(
+                PulsarConfigurationLoader.convertFrom(config), mock(PulsarResources.class));
+        PulsarWorkerService pulsarWorkerService = mock(PulsarWorkerService.class);
+        when(pulsarWorkerService.getWorkerConfig()).thenReturn(config);
+        when(pulsarWorkerService.getAuthorizationService()).thenReturn(authorizationService);
+        when(pulsarWorkerService.isInitialized()).thenReturn(true);
+        when(pulsarWorkerService.getMetricsGenerator()).thenReturn(mock(MetricsGenerator.class));
+        LeaderService leaderService = mock(LeaderService.class);
+        when(leaderService.isLeader()).thenReturn(true);
+        when(pulsarWorkerService.getLeaderService()).thenReturn(leaderService);
+        worker = new WorkerImpl(() -> pulsarWorkerService);
+    }
+
+    @Test(dataProvider = "authParamsForNonSuperusers")
+    public void ensureNonSuperuserCombinationsFailAuthorization(AuthenticationParameters authParams) throws Throwable {
+        assertThrowsRestException401(() -> worker.getCluster(authParams));
+        assertThrowsRestException401(() -> worker.getClusterLeader(authParams));
+        assertThrowsRestException401(() -> worker.getAssignments(authParams));
+        assertThrowsRestException401(() -> worker.getWorkerMetrics(authParams));
+        assertThrowsRestException401(() -> worker.getFunctionsMetrics(authParams));
+        assertThrowsRestException401(() -> worker.getListOfConnectors(authParams));
+        assertThrowsRestException401(() -> worker.rebalance(null, authParams));
+        assertThrowsRestException401(() -> worker.drain(null, "test", authParams, false));
+        assertThrowsRestException401(() -> worker.getDrainStatus(null, "test", authParams, false));
+        // This endpoint is not protected
+        assertTrue(worker.isLeaderReady(authParams));
+    }
+
+    private void assertThrowsRestException401(Assert.ThrowingRunnable runnable) throws Throwable {
+        try {
+            runnable.run();
+            fail("Should have thrown RestException");
+        } catch (RestException e) {
+            assertEquals(e.getResponse().getStatus(), 401);
+        }
+    }
+}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
index 80b626165a5d8..32a104c576993 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
@@ -53,6 +53,7 @@
 import org.apache.distributedlog.api.namespace.Namespace;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.Functions;
 import org.apache.pulsar.client.admin.Namespaces;
 import org.apache.pulsar.client.admin.PulsarAdmin;
@@ -542,7 +543,7 @@ private void testRegisterFunctionMissingArguments(
                     details,
                     functionPkgUrl,
                     JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
-                    null);
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
             throw new RuntimeException(e);
         }
@@ -560,7 +561,7 @@ private void registerDefaultFunction() {
                     mockedFormData,
                     null,
                     JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
-                    null);
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
             throw new RuntimeException(e);
         }
@@ -943,7 +944,7 @@ private void testUpdateFunctionMissingArguments(
                     details,
                     null,
                     JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
-                    null);
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
             throw new RuntimeException(e);
         }
@@ -971,7 +972,7 @@ private void updateDefaultFunction() {
                     mockedFormData,
                     null,
                     JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
-                    null);
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
             throw new RuntimeException(e);
         }
@@ -1048,7 +1049,7 @@ public void testUpdateFunctionWithUrl() throws Exception {
                     null,
                     filePackageUrl,
                     JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
-                    null);
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
             throw new RuntimeException(e);
         }
@@ -1144,7 +1145,7 @@ private void testDeregisterFunctionMissingArguments(
                 tenant,
                 namespace,
                 function,
-                null);
+                AuthenticationParameters.builder().build());
     }
 
     private void deregisterDefaultFunction() {
@@ -1152,7 +1153,7 @@ private void deregisterDefaultFunction() {
                 tenant,
                 namespace,
                 function,
-                null);
+                AuthenticationParameters.builder().build());
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
@@ -1257,7 +1258,8 @@ private void testGetFunctionMissingArguments(
         resource.getFunctionInfo(
                 tenant,
                 namespace,
-                function, null
+                function,
+                AuthenticationParameters.builder().build()
         );
 
     }
@@ -1266,7 +1268,8 @@ private FunctionDetails getDefaultFunctionInfo() throws IOException {
         String json = (String) resource.getFunctionInfo(
                 tenant,
                 namespace,
-                function, null
+                function,
+                AuthenticationParameters.builder().build()
         ).getEntity();
         FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
         mergeJson(json, functionDetailsBuilder);
@@ -1353,7 +1356,8 @@ private void testListFunctionsMissingArguments(
     ) {
         resource.listFunctions(
                 tenant,
-                namespace, null
+                namespace,
+                AuthenticationParameters.builder().build()
         );
 
     }
@@ -1361,7 +1365,8 @@ private void testListFunctionsMissingArguments(
     private List listDefaultFunctions() {
         return new Gson().fromJson((String) resource.listFunctions(
                 tenant,
-                namespace, null
+                namespace,
+                AuthenticationParameters.builder().build()
         ).getEntity(), List.class);
     }
 
@@ -1390,7 +1395,8 @@ public void testDownloadFunctionHttpUrl() throws Exception {
                 "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
         String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
         FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService);
-        StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl, null).getEntity();
+        StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl,
+                AuthenticationParameters.builder().build()).getEntity();
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1407,8 +1413,8 @@ public void testDownloadFunctionFile() throws Exception {
         String fileLocation = file.getAbsolutePath().replace('\\', '/');
         String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
         FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService);
-        StreamingOutput streamOutput =
-                (StreamingOutput) function.downloadFunction("file:///" + fileLocation, null).getEntity();
+        StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation,
+                        AuthenticationParameters.builder().build()).getEntity();
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1440,7 +1446,8 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception {
         functionConfig.setOutputSerdeClassName(outputSerdeClassName);
         try {
             resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl,
-                    JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), null);
+                    JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
            throw new RuntimeException(e);
         }
@@ -1474,7 +1481,8 @@ public void testRegisterFunctionWithConflictingFields() throws Exception {
         functionConfig.setOutputSerdeClassName(outputSerdeClassName);
         try {
             resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl,
-                    JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), null);
+                    JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
+                    AuthenticationParameters.builder().build());
         } catch (InvalidProtocolBufferException e) {
             throw new RuntimeException(e);
         }
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java
index c34b26180f876..337999bf2ad5e 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java
@@ -49,6 +49,7 @@
 import org.apache.distributedlog.api.namespace.Namespace;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.Functions;
 import org.apache.pulsar.client.admin.Namespaces;
 import org.apache.pulsar.client.admin.Packages;
@@ -571,7 +572,7 @@ private void testRegisterFunctionMissingArguments(
                 details,
                 functionPkgUrl,
                 functionConfig,
-                null, null);
+                null);
 
     }
 
@@ -585,7 +586,7 @@ public void testMissingFunctionConfig() {
                 mockedFormData,
                 null,
                 null,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided")
@@ -600,7 +601,7 @@ public void testUpdateMissingFunctionConfig() {
                 mockedFormData,
                 null,
                 null,
-                null, null, null);
+                null, null);
     }
 
     private void registerDefaultFunction() {
@@ -617,7 +618,7 @@ private void registerDefaultFunctionWithPackageUrl(String packageUrl) {
             mockedFormData,
             packageUrl,
             functionConfig,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists")
@@ -795,7 +796,7 @@ public void testRegisterFunctionSuccessK8sNoUpload() throws Exception {
                     mockedFormData,
                     null,
                     functionConfig,
-                    null, null);
+                    null);
         }
     }
 
@@ -852,7 +853,7 @@ public void testRegisterFunctionSuccessK8sWithUpload() throws Exception {
                         mockedFormData,
                         null,
                         functionConfig,
-                        null, null);
+                        null);
                 Assert.fail();
             } catch (RuntimeException e) {
                 Assert.assertEquals(e.getMessage(), injectedErrMsg);
@@ -1115,7 +1116,7 @@ private void testUpdateFunctionMissingArguments(
             details,
             null,
             functionConfig,
-                null, null, null);
+                null, null);
 
     }
 
@@ -1143,7 +1144,7 @@ private void updateDefaultFunctionWithPackageUrl(String packageUrl) {
             mockedFormData,
             packageUrl,
             functionConfig,
-                null, null, null);
+                null, null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
@@ -1217,7 +1218,7 @@ public void testUpdateFunctionWithUrl() {
             null,
             filePackageUrl,
             functionConfig,
-                null, null, null);
+                null, null);
 
     }
 
@@ -1331,7 +1332,7 @@ private void testDeregisterFunctionMissingArguments(
             tenant,
             namespace,
             function,
-                null, null);
+                null);
     }
 
     private void deregisterDefaultFunction() {
@@ -1339,7 +1340,7 @@ private void deregisterDefaultFunction() {
             tenant,
             namespace,
             function,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
@@ -1444,7 +1445,7 @@ private void testGetFunctionMissingArguments(
         resource.getFunctionInfo(
             tenant,
             namespace,
-            function,null,null
+            function,null
         );
 
     }
@@ -1454,7 +1455,6 @@ private FunctionConfig getDefaultFunctionInfo() {
             tenant,
             namespace,
             function,
-                null,
                 null
         );
     }
@@ -1539,7 +1539,7 @@ private void testListFunctionsMissingArguments(
     ) {
         resource.listFunctions(
             tenant,
-            namespace,null,null
+            namespace,null
         );
 
     }
@@ -1547,7 +1547,7 @@ private void testListFunctionsMissingArguments(
     private List listDefaultFunctions() {
         return resource.listFunctions(
             tenant,
-            namespace,null,null
+            namespace,null
         );
     }
 
@@ -1604,7 +1604,7 @@ public void testDownloadFunctionHttpUrl() throws Exception {
                 "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
         String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
 
-        StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null, null);
+        StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null);
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1619,7 +1619,7 @@ public void testDownloadFunctionFile() throws Exception {
         String fileLocation = file.getAbsolutePath().replace('\\', '/');
         String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
 
-        StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null, null);
+        StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null);
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1643,7 +1643,7 @@ public void testDownloadFunctionBuiltinConnector() throws Exception {
         when(connectorsManager.getConnector("cassandra")).thenReturn(connector);
         when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager);
 
-        StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null, null);
+        StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null);
 
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
@@ -1672,7 +1672,7 @@ public void testDownloadFunctionBuiltinFunction() throws Exception {
         when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class));
         when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager);
 
-        StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null, null);
+        StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null);
 
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
@@ -1707,7 +1707,8 @@ public void testDownloadFunctionBuiltinConnectorByName() throws Exception {
         when(connectorsManager.getConnector("cassandra")).thenReturn(connector);
         when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager);
 
-        StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, false);
+        StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function,
+                AuthenticationParameters.builder().build(), false);
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1740,7 +1741,8 @@ public void testDownloadFunctionBuiltinFunctionByName() throws Exception {
         when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class));
         when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager);
 
-        StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, false);
+        StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function,
+                AuthenticationParameters.builder().build(), false);
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1774,7 +1776,8 @@ public void testDownloadTransformFunctionByName() throws Exception {
         when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class));
         when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager);
 
-        StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, true);
+        StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function,
+                AuthenticationParameters.builder().build(), true);
         File pkgFile = new File(testDir, UUID.randomUUID().toString());
         OutputStream output = new FileOutputStream(pkgFile);
         streamOutput.write(output);
@@ -1804,7 +1807,7 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception {
         functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
         functionConfig.setOutput(outputTopic);
         functionConfig.setOutputSerdeClassName(outputSerdeClassName);
-        resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null, null);
+        resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null);
 
     }
 
@@ -1834,7 +1837,7 @@ public void testRegisterFunctionWithConflictingFields() throws Exception {
         functionConfig.setOutput(outputTopic);
         functionConfig.setOutputSerdeClassName(outputSerdeClassName);
         resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is either not set or cannot be determined")
@@ -1856,7 +1859,7 @@ public void testCreateFunctionWithoutSettingRuntime() throws Exception {
         functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
         functionConfig.setOutput(outputTopic);
         functionConfig.setOutputSerdeClassName(outputSerdeClassName);
-        resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null, null);
+        resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null);
 
     }
 
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java
index 9f0de2a12e091..9f786c5b73aac 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java
@@ -52,6 +52,7 @@
 import org.apache.distributedlog.api.namespace.Namespace;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.Functions;
 import org.apache.pulsar.client.admin.Namespaces;
 import org.apache.pulsar.client.admin.Packages;
@@ -534,7 +535,7 @@ private void testRegisterSinkMissingArguments(
                 details,
                 pkgUrl,
                 sinkConfig,
-                null, null);
+                null);
 
     }
 
@@ -548,7 +549,7 @@ public void testMissingSinkConfig() {
                 mockedFormData,
                 null,
                 null,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink config is not provided")
@@ -562,7 +563,7 @@ public void testUpdateMissingSinkConfig() {
                 mockedFormData,
                 null,
                 null,
-                null, null, null);
+                null, null);
     }
 
     private void registerDefaultSink() throws IOException {
@@ -580,7 +581,7 @@ private void registerDefaultSinkWithPackageUrl(String packageUrl) throws IOExcep
                     mockedFormData,
                     packageUrl,
                     sinkConfig,
-                    null, null);
+                    null);
         }
     }
 
@@ -652,7 +653,7 @@ public void testRegisterSinkConflictingFields() throws Exception {
                     mockedFormData,
                     null,
                     sinkConfig,
-                    null, null);
+                    null);
         }
     }
 
@@ -754,7 +755,7 @@ public void testRegisterSinkSuccessWithTransformFunction() throws Exception {
                     mockedFormData,
                     null,
                     sinkConfig,
-                    null, null);
+                    null);
         }
     }
 
@@ -803,7 +804,7 @@ public void testRegisterSinkFailureWithInvalidTransformFunction() throws Excepti
                         mockedFormData,
                         null,
                         sinkConfig,
-                        null, null);
+                        null);
             }
         } catch (RestException e) {
             // expected exception
@@ -1021,7 +1022,7 @@ private void testUpdateSinkMissingArguments(
                 details,
                 null,
                 sinkConfig,
-                null, null, null);
+                null, null);
 
     }
 
@@ -1064,7 +1065,7 @@ private void updateDefaultSinkWithPackageUrl(String packageUrl) throws Exception
                     mockedFormData,
                     packageUrl,
                     sinkConfig,
-                    null, null, null);
+                    null, null);
         }
     }
 
@@ -1149,7 +1150,7 @@ public void testUpdateSinkWithUrl() throws Exception {
                 null,
                 filePackageUrl,
                 sinkConfig,
-                null, null, null);
+                null, null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "sink failed to register")
@@ -1249,7 +1250,7 @@ public void testUpdateSinkDifferentTransformFunction() throws Exception {
                     mockedFormData,
                     null,
                     sinkConfig,
-                    null, null, null);
+                    null, null);
         }
     }
 
@@ -1308,7 +1309,7 @@ private void testDeregisterSinkMissingArguments(
                 tenant,
                 namespace,
                 sink,
-                null, null);
+                null);
 
     }
 
@@ -1317,7 +1318,7 @@ private void deregisterDefaultSink() {
                 tenant,
                 namespace,
                 sink,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink test-sink doesn't exist")
@@ -1532,7 +1533,8 @@ private void testGetSinkMissingArguments(
         resource.getFunctionInfo(
                 tenant,
                 namespace,
-                sink, null, null
+                sink,
+                AuthenticationParameters.builder().build()
         );
 
     }
@@ -1541,7 +1543,8 @@ private SinkConfig getDefaultSinkInfo() {
         return resource.getSinkInfo(
                 tenant,
                 namespace,
-                sink
+                sink,
+                AuthenticationParameters.builder().build()
         );
     }
 
@@ -1634,7 +1637,8 @@ private void testListSinksMissingArguments(
     ) {
         resource.listFunctions(
                 tenant,
-                namespace, null, null
+                namespace,
+                AuthenticationParameters.builder().build()
         );
 
     }
@@ -1642,7 +1646,8 @@ private void testListSinksMissingArguments(
     private List listDefaultSinks() {
         return resource.listFunctions(
                 tenant,
-                namespace, null, null
+                namespace,
+                AuthenticationParameters.builder().build()
         );
     }
 
@@ -1779,7 +1784,7 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception {
                     mockedFormData,
                     null,
                     sinkConfig,
-                    null, null);
+                    null);
         }
     }
 
@@ -1836,7 +1841,7 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception {
                         mockedFormData,
                         null,
                         sinkConfig,
-                        null, null);
+                        null);
                 Assert.fail();
             } catch (RuntimeException e) {
                 Assert.assertEquals(e.getMessage(), injectedErrMsg);
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java
index 303fa559b1d8e..36be2de716661 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java
@@ -51,6 +51,7 @@
 import org.apache.distributedlog.api.namespace.Namespace;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.client.admin.Functions;
 import org.apache.pulsar.client.admin.Namespaces;
 import org.apache.pulsar.client.admin.Packages;
@@ -488,7 +489,7 @@ private void testRegisterSourceMissingArguments(
                 details,
                 pkgUrl,
                 sourceConfig,
-                null, null);
+                null);
 
     }
 
@@ -502,7 +503,7 @@ public void testMissingSinkConfig() {
                 mockedFormData,
                 null,
                 null,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source config is not provided")
@@ -516,7 +517,7 @@ public void testUpdateMissingSinkConfig() {
                 mockedFormData,
                 null,
                 null,
-                null, null, null);
+                null, null);
     }
 
     private void registerDefaultSource() throws IOException {
@@ -533,7 +534,7 @@ private void registerDefaultSourceWithPackageUrl(String packageUrl) throws IOExc
                 null,
                 packageUrl,
                 sourceConfig,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source test-source already "
@@ -634,7 +635,7 @@ public void testRegisterSourceConflictingFields() throws Exception {
                     mockedFormData,
                     null,
                     sourceConfig,
-                    null, null);
+                    null);
         }
     }
 
@@ -937,7 +938,7 @@ private void testUpdateSourceMissingArguments(
                 details,
                 null,
                 sourceConfig,
-                null, null, null);
+                null, null);
 
     }
 
@@ -983,7 +984,7 @@ private void updateDefaultSourceWithPackageUrl(String packageUrl) throws Excepti
                     mockedFormData,
                     packageUrl,
                     sourceConfig,
-                    null, null, null);
+                    null, null);
         }
     }
 
@@ -1069,7 +1070,7 @@ public void testUpdateSourceWithUrl() throws Exception {
                 null,
                 filePackageUrl,
                 sourceConfig,
-                null, null, null);
+                null, null);
 
     }
 
@@ -1182,7 +1183,7 @@ private void testDeregisterSourceMissingArguments(
                 tenant,
                 namespace,
                 function,
-                null, null);
+                null);
 
     }
 
@@ -1191,7 +1192,7 @@ private void deregisterDefaultSource() {
                 tenant,
                 namespace,
                 source,
-                null, null);
+                null);
     }
 
     @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source test-source doesn't " +
@@ -1385,7 +1386,8 @@ private void testGetSourceMissingArguments(
         resource.getFunctionInfo(
                 tenant,
                 namespace,
-                source, null, null
+                source,
+                AuthenticationParameters.builder().build()
         );
     }
 
@@ -1393,7 +1395,8 @@ private SourceConfig getDefaultSourceInfo() {
         return resource.getSourceInfo(
                 tenant,
                 namespace,
-                source
+                source,
+                AuthenticationParameters.builder().build()
         );
     }
 
@@ -1476,14 +1479,17 @@ private void testListSourcesMissingArguments(
     ) {
         resource.listFunctions(
                 tenant,
-                namespace, null, null
+                namespace,
+                AuthenticationParameters.builder().build()
         );
     }
 
     private List listDefaultSources() {
         return resource.listFunctions(
                 tenant,
-                namespace, null, null);
+                namespace,
+                AuthenticationParameters.builder().build()
+        );
     }
 
     @Test

From 8084c8f462f4848190c6925f77efe6fa685a8c02 Mon Sep 17 00:00:00 2001
From: csthomas1 
Date: Fri, 7 Apr 2023 18:50:14 +0000
Subject: [PATCH 152/404] Added KubernetesRuntime for issue#19583 to verify
 function tenant, namespace, name translation to k8s labels

---
 .../kubernetes/KubernetesRuntimeTest.java     | 67 +++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index 1e8194eed443a..15bf4f2e8c97b 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -24,7 +24,11 @@
 import com.google.gson.JsonObject;
 import com.google.protobuf.util.JsonFormat;
 import io.kubernetes.client.custom.Quantity;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.AppsV1Api;
+import io.kubernetes.client.openapi.apis.CoreV1Api;
 import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1PodList;
 import io.kubernetes.client.openapi.models.V1PodSpec;
 import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
 import io.kubernetes.client.openapi.models.V1ResourceRequirements;
@@ -58,6 +62,7 @@
 
 import java.lang.reflect.Type;
 import java.math.BigDecimal;
+import java.net.HttpURLConnection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -65,11 +70,23 @@
 import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import okhttp3.Call;
+import okhttp3.Response;
 
 import static org.apache.pulsar.functions.runtime.RuntimeUtils.FUNCTIONS_INSTANCE_CLASSPATH;
 import static org.apache.pulsar.functions.utils.FunctionCommon.roundDecimal;
+import org.mockito.ArgumentMatcher;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotEquals;
@@ -1233,4 +1250,54 @@ private void assertMetricsPortConfigured(Map functionRuntimeFact
                     .contains("--metrics_port 0"));
         }
     }
+    
+    @Test
+    public void testDeleteStatefulSetWithTranslatedKubernetesLabelChars() throws Exception {
+        InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false);
+        config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false, (fb) -> {
+            return fb.setTenant("c:tenant").setNamespace("c:ns").setName("c:fn");
+        }));
+
+        CoreV1Api coreApi = mock(CoreV1Api.class);
+        AppsV1Api appsApi = mock(AppsV1Api.class);
+        
+        Call successfulCall = mock(Call.class);
+        Response okResponse = mock(Response.class);
+        when(okResponse.code()).thenReturn(HttpURLConnection.HTTP_OK);
+        when(okResponse.isSuccessful()).thenReturn(true);
+        when(okResponse.message()).thenReturn("");
+        when(successfulCall.execute()).thenReturn(okResponse);
+        
+        final String expectedFunctionNamePrefix = String.format("pf-%s-%s-%s", "c-tenant", "c-ns", "c-fn");
+        
+        factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0);
+        factory.setCoreClient(coreApi);
+        factory.setAppsClient(appsApi);
+
+        ArgumentMatcher hasTranslatedFunctionName = (String t) -> {
+            return t.startsWith(expectedFunctionNamePrefix);
+        };
+        
+        when(appsApi.deleteNamespacedStatefulSetCall(
+                argThat(hasTranslatedFunctionName),
+                anyString(), isNull(), isNull(), anyInt(), isNull(), anyString(), any(), isNull())).thenReturn(successfulCall);
+
+        ApiException notFoundException = mock(ApiException.class);
+        when(notFoundException.getCode()).thenReturn(HttpURLConnection.HTTP_NOT_FOUND);
+        when(appsApi.readNamespacedStatefulSet(argThat(hasTranslatedFunctionName), anyString(),
+                isNull(), isNull(), isNull())).thenThrow(notFoundException);
+
+        V1PodList podList = mock(V1PodList.class);
+        when(podList.getItems()).thenReturn(Collections.emptyList());
+        
+        String expectedLabels = String.format("tenant=%s,namespace=%s,name=%s", "c-tenant", "c-ns", "c-fn");
+        
+        when(coreApi.listNamespacedPod(anyString(), isNull(), isNull(), isNull(), isNull(),
+                eq(expectedLabels), isNull(), isNull(), isNull(), isNull(), isNull())).thenReturn(podList);
+        KubernetesRuntime kr = factory.createContainer(config, "/test/code", "code.yml", "/test/transforms", "transform.yml", Long.MIN_VALUE);        
+        kr.deleteStatefulSet();
+        
+        verify(coreApi).listNamespacedPod(anyString(), isNull(), isNull(), isNull(), isNull(),
+                eq(expectedLabels), isNull(), isNull(), isNull(), isNull(), isNull());
+    }
 }

From 421d707419ee42378b9c5b37812bd5c617ef2c9a Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Sat, 8 Apr 2023 03:49:16 -0700
Subject: [PATCH 153/404] [improve][broker] PIP-192 made split handler
 idempotent (#19988)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement: https://github.com/apache/pulsar/issues/16691

### Modifications
This PR
- Makes split handler idempotent .
- Makes Leader's orphan monitor keep trying to send split msg until finished.
- Select bundle boundaries at the SplitScheduler to have the same split boundaries for each Split handler retry.
- Adds a split condition to check if the parent's Splitting state has moved.
- Made Admin Unload command forceful to unload any bundles in invalid states.
---
 conf/broker.conf                              |   2 +-
 .../pulsar/broker/ServiceConfiguration.java   |   2 +-
 .../extensions/ExtensibleLoadManagerImpl.java |   2 +-
 .../channel/ServiceUnitStateChannelImpl.java  | 263 +++++++-----
 .../channel/ServiceUnitStateData.java         |  15 +
 .../loadbalance/extensions/models/Split.java  |   6 +-
 .../loadbalance/extensions/models/Unload.java |  10 +-
 ...faultNamespaceBundleSplitStrategyImpl.java |  76 +++-
 .../broker/namespace/NamespaceService.java    |  10 +-
 .../common/naming/NamespaceBundleFactory.java |   8 +-
 .../naming/NamespaceBundleSplitAlgorithm.java |   3 +
 ...pecifiedPositionsBundleSplitAlgorithm.java |  47 ++-
 .../ExtensibleLoadManagerImplTest.java        |   1 +
 .../channel/ServiceUnitStateChannelTest.java  | 399 +++++++++++++++---
 .../extensions/channel/models/SplitTest.java  |   8 +-
 .../scheduler/SplitSchedulerTest.java         |  19 +-
 ...faultNamespaceBundleSplitStrategyTest.java |  81 +++-
 ...fiedPositionsBundleSplitAlgorithmTest.java |  15 +
 18 files changed, 767 insertions(+), 200 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 1bbb12313f9a6..414756abcb7cb 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1455,7 +1455,7 @@ loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3
 # the service-unit state channel when there are a large number of bundles.
 # minimum value = 30 secs
 # (only used in load balancer extension logics)
-loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds=604800
+loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds=3600
 
 
 ### --- Replication --- ###
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 52cebe15f6a5c..c86358b9b5b9e 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2609,7 +2609,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "minimum value = 30 secs"
                     + "(only used in load balancer extension logics)"
     )
-    private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800;
+    private long loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds = 3600;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index c1234b6dab21e..e054a1bc662c7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -449,7 +449,7 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                         log.warn(msg);
                         throw new IllegalArgumentException(msg);
                     }
-                    Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker);
+                    Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker, true);
                     UnloadDecision unloadDecision =
                             new UnloadDecision(unload, UnloadDecision.Label.Success, UnloadDecision.Reason.Admin);
                     return unloadAsync(unloadDecision,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index ec79698db1f05..bd62c53be6083 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -46,7 +46,6 @@
 import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -61,6 +60,7 @@
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -200,7 +200,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) {
                 CompletableFuture>newBuilder().build();
         this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build();
         this.stateChangeListeners = new StateChangeListeners();
-        this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateCleanUpDelayTimeInSeconds()
+        this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateTombstoneDelayTimeInSeconds()
                 * 1000;
         this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS;
         this.ownershipMonitorDelayTimeInSecs = OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS;
@@ -563,10 +563,11 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) {
         ServiceUnitStateData next;
         if (isTransferCommand(unload)) {
             next = new ServiceUnitStateData(
-                    Releasing, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit));
+                    Releasing, unload.destBroker().get(), unload.sourceBroker(),
+                    unload.force(), getNextVersionId(serviceUnit));
         } else {
             next = new ServiceUnitStateData(
-                    Releasing, null, unload.sourceBroker(), getNextVersionId(serviceUnit));
+                    Releasing, null, unload.sourceBroker(), unload.force(), getNextVersionId(serviceUnit));
         }
         return pubAsync(serviceUnit, next).whenComplete((__, ex) -> {
             if (ex != null) {
@@ -660,7 +661,7 @@ private void log(Throwable e, String serviceUnit, ServiceUnitStateData data, Ser
                 long handlerTotalCount = getHandlerTotalCounter(data).get();
                 long handlerFailureCount = getHandlerFailureCounter(data).get();
                 log.info("{} handled {} event for serviceUnit:{}, cur:{}, next:{}, "
-                                + "totalHandledRequests{}, totalFailedRequests:{}",
+                                + "totalHandledRequests:{}, totalFailedRequests:{}",
                         lookupServiceAddress, getLogEventTag(data), serviceUnit,
                         data == null ? "" : data,
                         next == null ? "" : next,
@@ -671,7 +672,7 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit,
             long handlerTotalCount = getHandlerTotalCounter(data).get();
             long handlerFailureCount = getHandlerFailureCounter(data).incrementAndGet();
             log.error("{} failed to handle {} event for serviceUnit:{}, cur:{}, next:{}, "
-                            + "totalHandledRequests{}, totalFailedRequests:{}",
+                            + "totalHandledRequests:{}, totalFailedRequests:{}",
                     lookupServiceAddress, getLogEventTag(data), serviceUnit,
                     data == null ? "" : data,
                     next == null ? "" : next,
@@ -857,110 +858,158 @@ private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnit
                 boundariesSet.add(subBundle.getKeyRange().upperEndpoint());
             });
             boundaries = new ArrayList<>(boundariesSet);
-            nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_ALGO;
+            nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_FORCE_ALGO;
         }
         final AtomicInteger counter = new AtomicInteger(0);
+        var childBundles = data.splitServiceUnitToDestBroker().keySet().stream()
+                .map(child -> bundleFactory.getBundle(
+                        bundle.getNamespaceObject().toString(), child))
+                .collect(Collectors.toList());
         this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, nsBundleSplitAlgorithm,
-                bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture);
+                bundle, childBundles, boundaries, data, counter, startTime, completionFuture);
         return completionFuture;
     }
 
+
+
     @VisibleForTesting
     protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                                                 NamespaceBundleFactory bundleFactory,
                                                 NamespaceBundleSplitAlgorithm algorithm,
-                                                NamespaceBundle bundle,
+                                                NamespaceBundle parentBundle,
+                                                List childBundles,
                                                 List boundaries,
-                                                String serviceUnit,
-                                                ServiceUnitStateData data,
+                                                ServiceUnitStateData parentData,
                                                 AtomicInteger counter,
                                                 long startTime,
                                                 CompletableFuture completionFuture) {
-        CompletableFuture> updateFuture = new CompletableFuture<>();
-
-        namespaceService.getSplitBoundary(bundle, algorithm, boundaries)
-                .thenAccept(splitBundlesPair -> {
-            // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper.
-            if (splitBundlesPair == null) {
-                String msg = format("Bundle %s not found under namespace", serviceUnit);
-                updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg));
-                return;
-            }
-            ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.sourceBroker(), VERSION_ID_INIT);
-            NamespaceBundles targetNsBundle = splitBundlesPair.getLeft();
-            List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight());
-            List successPublishedBundles =
-                    Collections.synchronizedList(new ArrayList<>(splitBundles.size()));
-            List> futures = new ArrayList<>(splitBundles.size());
-            for (NamespaceBundle sBundle : splitBundles) {
-                futures.add(pubAsync(sBundle.toString(), next).thenAccept(__ -> successPublishedBundles.add(sBundle)));
-            }
-            NamespaceName nsname = bundle.getNamespaceObject();
-            FutureUtil.waitForAll(futures)
-                    .thenCompose(__ -> namespaceService.updateNamespaceBundles(nsname, targetNsBundle))
-                    .thenCompose(__ -> namespaceService.updateNamespaceBundlesForPolicies(nsname, targetNsBundle))
-                    .thenRun(() -> {
-                        bundleFactory.invalidateBundleCache(bundle.getNamespaceObject());
-                        updateFuture.complete(splitBundles);
-                    }).exceptionally(e -> {
-                        // Clean the new bundle when has exception.
-                        List> futureList = new ArrayList<>();
-                        for (NamespaceBundle sBundle : successPublishedBundles) {
-                            futureList.add(tombstoneAsync(sBundle.toString()).thenAccept(__ -> {}));
-                        }
-                        FutureUtil.waitForAll(futureList)
-                                .whenComplete((__, ex) -> {
-                                    if (ex != null) {
-                                        log.warn("Clean new bundles failed,", ex);
-                                    }
-                                    updateFuture.completeExceptionally(e);
-                                });
-                        return null;
-                    });
-        }).exceptionally(e -> {
-            updateFuture.completeExceptionally(e);
-            return null;
-        });
+        ownChildBundles(childBundles, parentData)
+                .thenCompose(__ -> getSplitNamespaceBundles(
+                        namespaceService, bundleFactory, algorithm, parentBundle, childBundles, boundaries))
+                .thenCompose(namespaceBundles -> updateSplitNamespaceBundlesAsync(
+                        namespaceService, bundleFactory, parentBundle, namespaceBundles))
+                .thenAccept(__ -> // Update bundled_topic cache for load-report-generation
+                        pulsar.getBrokerService().refreshTopicToStatsMaps(parentBundle))
+                .thenAccept(__ -> pubAsync(parentBundle.toString(), new ServiceUnitStateData(
+                                Deleted, null, parentData.sourceBroker(), getNextVersionId(parentData))))
+                .thenAccept(__ -> {
+                    double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime));
+                    log.info("Successfully split {} parent namespace-bundle to {} in {} ms",
+                            parentBundle, childBundles, splitBundleTime);
+                    completionFuture.complete(null);
+                })
+                .exceptionally(ex -> {
+                    // Retry several times on BadVersion
+                    Throwable throwable = FutureUtil.unwrapCompletionException(ex);
+                    if ((throwable instanceof MetadataStoreException.BadVersionException)
+                            && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) {
+                        log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit",
+                                counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex);
+                        pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(
+                                namespaceService, bundleFactory, algorithm, parentBundle, childBundles,
+                                        boundaries, parentData, counter, startTime, completionFuture),
+                                100, MILLISECONDS);
+                    } else {
+                        // Retry enough, or meet other exception
+                        String msg = format("Failed to split bundle %s, Retried %d th / %d limit, reason %s",
+                                parentBundle.toString(), counter.get(),
+                                NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, throwable.getMessage());
+                        log.warn(msg, throwable);
+                        completionFuture.completeExceptionally(
+                                new BrokerServiceException.ServiceUnitNotReadyException(msg));
+                    }
+                    return null;
+                });
+    }
 
-        updateFuture.thenAccept(r -> {
-            // Delete the old bundle
-            pubAsync(serviceUnit, new ServiceUnitStateData(
-                    Deleted, null, data.sourceBroker(), getNextVersionId(data)))
-                    .thenRun(() -> {
-                // Update bundled_topic cache for load-report-generation
-                pulsar.getBrokerService().refreshTopicToStatsMaps(bundle);
-                // TODO: Update the load data immediately if needed.
-                completionFuture.complete(null);
-                double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime));
-                log.info("Successfully split {} parent namespace-bundle to {} in {} ms", serviceUnit, r,
-                        splitBundleTime);
-            }).exceptionally(e -> {
-                double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime));
-                String msg = format("Failed to free bundle %s in %s ms, under namespace [%s] with error %s",
-                        bundle.getNamespaceObject().toString(), splitBundleTime, bundle, e.getMessage());
-                completionFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg));
-                return null;
-            });
-        }).exceptionally(ex -> {
-            // Retry several times on BadVersion
-            Throwable throwable = FutureUtil.unwrapCompletionException(ex);
-            if ((throwable instanceof MetadataStoreException.BadVersionException)
-                    && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) {
-                log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit",
-                        counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex);
-                pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory,
-                        algorithm, bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture),
-                        100, MILLISECONDS);
-            } else if (throwable instanceof IllegalArgumentException) {
-                completionFuture.completeExceptionally(throwable);
+    private CompletableFuture ownChildBundles(List childBundles,
+                                                    ServiceUnitStateData parentData) {
+        List> futures = new ArrayList<>(childBundles.size());
+        var debug = debug();
+        for (var childBundle : childBundles) {
+            var childBundleStr = childBundle.toString();
+            var childData = tableview.get(childBundleStr);
+            if (childData != null) {
+                if (debug) {
+                    log.info("Already owned child bundle:{}", childBundleStr);
+                }
             } else {
-                // Retry enough, or meet other exception
-                String msg = format("Bundle: %s not success update nsBundles, counter %d, reason %s",
-                        bundle.toString(), counter.get(), throwable.getMessage());
-                completionFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg));
+                childData = new ServiceUnitStateData(Owned, parentData.sourceBroker(),
+                        VERSION_ID_INIT);
+                futures.add(pubAsync(childBundleStr, childData).thenApply(__ -> null));
             }
-            return null;
-        });
+        }
+
+        if (!futures.isEmpty()) {
+            return FutureUtil.waitForAll(futures);
+        } else {
+            return CompletableFuture.completedFuture(null);
+        }
+    }
+
+    private CompletableFuture getSplitNamespaceBundles(NamespaceService namespaceService,
+                                                                         NamespaceBundleFactory bundleFactory,
+                                                                         NamespaceBundleSplitAlgorithm algorithm,
+                                                                         NamespaceBundle parentBundle,
+                                                                         List childBundles,
+                                                                         List boundaries) {
+        CompletableFuture future = new CompletableFuture();
+        final var debug = debug();
+        var targetNsBundle = bundleFactory.getBundles(parentBundle.getNamespaceObject());
+        boolean found = false;
+        try {
+            targetNsBundle.validateBundle(parentBundle);
+        } catch (IllegalArgumentException e) {
+            if (debug) {
+                log.info("Namespace bundles do not contain the parent bundle:{}",
+                        parentBundle);
+            }
+            for (var childBundle : childBundles) {
+                try {
+                    targetNsBundle.validateBundle(childBundle);
+                    if (debug) {
+                        log.info("Namespace bundles contain the child bundle:{}",
+                                childBundle);
+                    }
+                } catch (Exception ex) {
+                    future.completeExceptionally(
+                            new BrokerServiceException.ServiceUnitNotReadyException(
+                                    "Namespace bundles do not contain the child bundle:" + childBundle, e));
+                    return future;
+                }
+            }
+            found = true;
+        } catch (Exception e) {
+            future.completeExceptionally(
+                    new BrokerServiceException.ServiceUnitNotReadyException(
+                            "Failed to validate the parent bundle in the namespace bundles.", e));
+            return future;
+        }
+        if (found) {
+            future.complete(targetNsBundle);
+            return future;
+        } else {
+            return namespaceService.getSplitBoundary(parentBundle, algorithm, boundaries)
+                    .thenApply(splitBundlesPair -> splitBundlesPair.getLeft());
+        }
+    }
+
+    private CompletableFuture updateSplitNamespaceBundlesAsync(
+            NamespaceService namespaceService,
+            NamespaceBundleFactory bundleFactory,
+            NamespaceBundle parentBundle,
+            NamespaceBundles splitNamespaceBundles) {
+        var namespaceName = parentBundle.getNamespaceObject();
+        return namespaceService.updateNamespaceBundles(
+                        namespaceName, splitNamespaceBundles)
+                .thenCompose(__ -> namespaceService.updateNamespaceBundlesForPolicies(
+                        namespaceName, splitNamespaceBundles))
+                .thenAccept(__ -> {
+                    bundleFactory.invalidateBundleCache(parentBundle.getNamespaceObject());
+                    if (debug()) {
+                        log.info("Successfully updated split namespace bundles and namespace bundle cache.");
+                    }
+                });
     }
 
     public void handleMetadataSessionEvent(SessionEvent e) {
@@ -1053,11 +1102,23 @@ private void scheduleCleanup(String broker, long delayInSecs) {
                 broker, delayInSecs, cleanupJobs.size());
     }
 
+
+    private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData,
+                                                                    String selectedBroker) {
+        if (orphanData.state() == Splitting) {
+            return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker,
+                    Map.copyOf(orphanData.splitServiceUnitToDestBroker()),
+                    true, getNextVersionId(orphanData));
+        } else {
+            return new ServiceUnitStateData(Owned, selectedBroker, true, getNextVersionId(orphanData));
+        }
+    }
+
     private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) {
+
         Optional selectedBroker = selectBroker(serviceUnit);
         if (selectedBroker.isPresent()) {
-            var override =
-                    new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData));
+            var override = getOverrideInactiveBrokerStateData(orphanData, selectedBroker.get());
             log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}",
                     serviceUnit, orphanData, override);
             publishOverrideEventAsync(serviceUnit, orphanData, override)
@@ -1157,7 +1218,13 @@ private Optional getOverrideInFlightStateData(
             case Assigning: {
                 return getRollForwardStateData(serviceUnit, nextVersionId);
             }
-            case Splitting, Releasing: {
+            case Splitting: {
+                return Optional.of(new ServiceUnitStateData(Splitting,
+                        orphanData.dstBroker(), orphanData.sourceBroker(),
+                        Map.copyOf(orphanData.splitServiceUnitToDestBroker()),
+                        true, nextVersionId));
+            }
+            case Releasing: {
                 if (availableBrokers.contains(orphanData.sourceBroker())) {
                     // rollback to the src
                     return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId));
@@ -1219,7 +1286,6 @@ protected void monitorOwnerships(List brokers) {
                     inactiveBrokers.add(dstBroker);
                 } else if (isInFlightState(state)
                         && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) {
-                    log.warn("Found orphan serviceUnit:{}, stateData:{}", serviceUnit, stateData);
                     orphanServiceUnits.put(serviceUnit, stateData);
                 }
             } else if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) {
@@ -1249,6 +1315,9 @@ protected void monitorOwnerships(List brokers) {
                 var overrideData = getOverrideInFlightStateData(
                         orphanServiceUnit, orphanData, activeBrokers);
                 if (overrideData.isPresent()) {
+                    log.info("Overriding in-flight state ownership serviceUnit:{} "
+                                    + "from orphanData:{} to overrideData:{}",
+                            orphanServiceUnit, orphanData, overrideData);
                     publishOverrideEventAsync(orphanServiceUnit, orphanData, overrideData.get())
                             .whenComplete((__, e) -> {
                                 if (e != null) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
index 52c7e27d65033..307d3a4acb175 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
@@ -45,10 +45,25 @@ public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sou
                 System.currentTimeMillis(), versionId);
     }
 
+    public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker,
+                                Map> splitServiceUnitToDestBroker, boolean force,
+                                long versionId) {
+        this(state, dstBroker, sourceBroker, splitServiceUnitToDestBroker, force,
+                System.currentTimeMillis(), versionId);
+    }
+
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) {
         this(state, dstBroker, sourceBroker, null, false, System.currentTimeMillis(), versionId);
     }
 
+    public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, boolean force,
+                                long versionId) {
+        this(state, dstBroker, sourceBroker, null, force,
+                System.currentTimeMillis(), versionId);
+    }
+
+
+
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) {
         this(state, dstBroker, null, null, false, System.currentTimeMillis(), versionId);
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
index ac9a36e6dbfd3..690fac59bc99c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
@@ -30,12 +30,8 @@ public record Split(
 
     public Split {
         Objects.requireNonNull(serviceUnit);
-        if (splitServiceUnitToDestBroker != null && splitServiceUnitToDestBroker.size() != 2) {
+        if (splitServiceUnitToDestBroker == null || splitServiceUnitToDestBroker.size() != 2) {
             throw new IllegalArgumentException("Split service unit should be split into 2 service units.");
         }
     }
-
-    public Split(String serviceUnit, String sourceBroker) {
-        this(serviceUnit, sourceBroker, null);
-    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java
index d474011919d97..753f8a942ced0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Unload.java
@@ -24,12 +24,18 @@
 /**
  * Defines the information required to unload or transfer a service unit(e.g. bundle).
  */
-public record Unload(String sourceBroker, String serviceUnit, Optional destBroker) {
+public record Unload(String sourceBroker, String serviceUnit, Optional destBroker, boolean force) {
     public Unload {
         Objects.requireNonNull(sourceBroker);
         Objects.requireNonNull(serviceUnit);
     }
+
+
     public Unload(String sourceBroker, String serviceUnit) {
-        this(sourceBroker, serviceUnit, Optional.empty());
+        this(sourceBroker, serviceUnit, Optional.empty(), false);
+    }
+
+    public Unload(String sourceBroker, String serviceUnit, Optional destBroker) {
+        this(sourceBroker, serviceUnit, destBroker, false);
     }
 }
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
index bd8b3aa66542a..7875c07b1224a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -26,18 +26,23 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 
@@ -53,12 +58,14 @@ public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleS
     private final Set decisionCache;
     private final Map namespaceBundleCount;
     private final Map splitConditionHitCounts;
+    private final Map splittingBundles;
     private final SplitCounter counter;
 
     public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) {
         decisionCache = new HashSet<>();
         namespaceBundleCount = new HashMap<>();
         splitConditionHitCounts = new HashMap<>();
+        splittingBundles = new HashMap<>();
         this.counter = counter;
 
     }
@@ -67,6 +74,7 @@ public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) {
     public Set findBundlesToSplit(LoadManagerContext context, PulsarService pulsar) {
         decisionCache.clear();
         namespaceBundleCount.clear();
+        splittingBundles.clear();
         final ServiceConfiguration conf = pulsar.getConfiguration();
         int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
         long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
@@ -78,6 +86,15 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
         boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled();
         var channel = ServiceUnitStateChannelImpl.get(pulsar);
 
+        for (var etr : channel.getOwnershipEntrySet()) {
+            var eData = etr.getValue();
+            if (eData.state() == ServiceUnitState.Splitting) {
+                String bundle = etr.getKey();
+                final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
+                splittingBundles.put(bundle, bundleRange);
+            }
+        }
+
         Map bundleStatsMap = pulsar.getBrokerService().getBundleStats();
         NamespaceBundleFactory namespaceBundleFactory =
                 pulsar.getNamespaceService().getNamespaceBundleFactory();
@@ -177,6 +194,27 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
                 continue;
             }
 
+            var ranges = bundleRange.split("_");
+            var foundSplittingBundle = false;
+            for (var etr : splittingBundles.entrySet()) {
+                var splittingBundle = etr.getKey();
+                if (splittingBundle.startsWith(namespace)) {
+                    var splittingBundleRange = etr.getValue();
+                    if (splittingBundleRange.startsWith(ranges[0])
+                            || splittingBundleRange.endsWith(ranges[1])) {
+                        if (debug) {
+                            log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                                    + " (parent) bundle:%s is in Splitting state.", bundle, splittingBundle));
+                        }
+                        foundSplittingBundle = true;
+                        break;
+                    }
+                }
+            }
+            if (foundSplittingBundle) {
+                continue;
+            }
+
             if (debug) {
                 log.info(String.format(
                         "Splitting bundle: %s. "
@@ -193,7 +231,43 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
                 ));
             }
             var decision = new SplitDecision();
-            decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
+            var namespaceService = pulsar.getNamespaceService();
+            var namespaceBundle = namespaceService.getNamespaceBundleFactory()
+                    .getBundle(namespaceName, bundleRange);
+            NamespaceBundleSplitAlgorithm algorithm =
+                    namespaceService.getNamespaceBundleSplitAlgorithmByName(
+                            conf.getDefaultNamespaceBundleSplitAlgorithm());
+            List splitBoundary = null;
+            try {
+                splitBoundary = namespaceService
+                        .getSplitBoundary(namespaceBundle,  null, algorithm)
+                        .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
+            } catch (Throwable e) {
+                counter.update(Failure, Unknown);
+                log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " Failed to get split boundaries.", bundle, e));
+                continue;
+            }
+            if (splitBoundary == null) {
+                counter.update(Failure, Unknown);
+                log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " The split boundaries is null.", bundle));
+                continue;
+            }
+            if (splitBoundary.size() != 1) {
+                counter.update(Failure, Unknown);
+                log.warn(String.format(CANNOT_SPLIT_BUNDLE_MSG + " The size of split boundaries is not 1. "
+                        + "splitBoundary:%s", bundle, splitBoundary));
+                continue;
+            }
+
+            var parentRange = namespaceBundle.getKeyRange();
+            var leftChildBundle = namespaceBundleFactory.getBundle(namespaceBundle.getNamespaceObject(),
+                    NamespaceBundleFactory.getRange(parentRange.lowerEndpoint(), splitBoundary.get(0)));
+            var rightChildBundle = namespaceBundleFactory.getBundle(namespaceBundle.getNamespaceObject(),
+                    NamespaceBundleFactory.getRange(splitBoundary.get(0), parentRange.upperEndpoint()));
+            Map> splitServiceUnitToDestBroker = Map.of(
+                    leftChildBundle.getBundleRange(), Optional.empty(),
+                    rightChildBundle.getBundleRange(), Optional.empty());
+            decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), splitServiceUnitToDestBroker));
             decision.succeed(reason);
             decisionCache.add(decision);
             int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index d092ef04018c8..c9cd33d6d87c8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -967,9 +967,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle,
      */
     public CompletableFuture>> getSplitBoundary(
             NamespaceBundle bundle, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm, List boundaries) {
-        BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config);
-        CompletableFuture> splitBoundary =
-                nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption);
+        CompletableFuture> splitBoundary = getSplitBoundary(bundle, boundaries, nsBundleSplitAlgorithm);
         return splitBoundary.thenCompose(splitBoundaries -> {
                     if (splitBoundaries == null || splitBoundaries.size() == 0) {
                         LOG.info("[{}] No valid boundary found in {} to split bundle {}",
@@ -981,6 +979,12 @@ public CompletableFuture>> getSplit
                 });
     }
 
+    public CompletableFuture> getSplitBoundary(
+            NamespaceBundle bundle, List boundaries, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm) {
+        BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config);
+        return nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption);
+    }
+
     private BundleSplitOption getBundleSplitOption(NamespaceBundle bundle,
                                                    List boundaries,
                                                    ServiceConfiguration config) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java
index f0f7deb940da1..937d2763767b6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java
@@ -272,8 +272,7 @@ public NamespaceBundle getBundle(String namespace, String bundleRange) {
         String[] boundaries = bundleRange.split("_");
         Long lowerEndpoint = Long.decode(boundaries[0]);
         Long upperEndpoint = Long.decode(boundaries[1]);
-        Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint,
-                (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
+        Range hashRange = getRange(lowerEndpoint, upperEndpoint);
         return getBundle(NamespaceName.get(namespace), hashRange);
     }
 
@@ -414,4 +413,9 @@ public static String getNamespaceFromPoliciesPath(String path) {
         return Joiner.on("/").join(i);
     }
 
+    public static Range getRange(Long lowerEndpoint, Long upperEndpoint) {
+        return Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint,
+                (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
+    }
+
 }
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java
index 01a61d8166563..1fd6fbcd6ea12 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleSplitAlgorithm.java
@@ -39,6 +39,9 @@ public interface NamespaceBundleSplitAlgorithm {
     NamespaceBundleSplitAlgorithm TOPIC_COUNT_EQUALLY_DIVIDE_ALGO = new TopicCountEquallyDivideBundleSplitAlgorithm();
     NamespaceBundleSplitAlgorithm SPECIFIED_POSITIONS_DIVIDE_ALGO =
             new SpecifiedPositionsBundleSplitAlgorithm();
+
+    NamespaceBundleSplitAlgorithm SPECIFIED_POSITIONS_DIVIDE_FORCE_ALGO =
+            new SpecifiedPositionsBundleSplitAlgorithm(true);
     NamespaceBundleSplitAlgorithm FLOW_OR_QPS_EQUALLY_DIVIDE_ALGO = new FlowOrQpsEquallyDivideBundleSplitAlgorithm();
 
     static NamespaceBundleSplitAlgorithm of(String algorithmName) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
index 1fd9a85cfa782..4a5f8a831c269 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
@@ -29,6 +29,16 @@
  * This algorithm divides the bundle into several parts by the specified positions.
  */
 public class SpecifiedPositionsBundleSplitAlgorithm implements NamespaceBundleSplitAlgorithm{
+
+    private boolean force;
+
+    public SpecifiedPositionsBundleSplitAlgorithm() {
+        force = false;
+    }
+
+    public SpecifiedPositionsBundleSplitAlgorithm(boolean force) {
+        this.force = force;
+    }
     @Override
     public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSplitOption) {
         NamespaceService service = bundleSplitOption.getService();
@@ -39,19 +49,28 @@ public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSp
         }
         // sort all positions
         Collections.sort(positions);
-        return service.getOwnedTopicListForNamespaceBundle(bundle).thenCompose(topics -> {
-            if (topics == null || topics.size() <= 1) {
-                return CompletableFuture.completedFuture(null);
-            }
-            List splitBoundaries = positions
-                    .stream()
-                    .filter(position -> position > bundle.getLowerEndpoint() && position < bundle.getUpperEndpoint())
-                    .collect(Collectors.toList());
-
-            if (splitBoundaries.size() == 0) {
-                return CompletableFuture.completedFuture(null);
-            }
-            return CompletableFuture.completedFuture(splitBoundaries);
-        });
+        if (force) {
+            return getBoundaries(bundle, positions);
+        } else {
+            return service.getOwnedTopicListForNamespaceBundle(bundle)
+                    .thenCompose(topics -> {
+                        if (topics == null || topics.size() <= 1) {
+                            return CompletableFuture.completedFuture(null);
+                        }
+                        return getBoundaries(bundle, positions);
+                    });
+        }
+    }
+
+    private CompletableFuture> getBoundaries(NamespaceBundle bundle, List positions) {
+        List splitBoundaries = positions
+                .stream()
+                .filter(position -> position > bundle.getLowerEndpoint() && position < bundle.getUpperEndpoint())
+                .collect(Collectors.toList());
+
+        if (splitBoundaries.size() == 0) {
+            return CompletableFuture.completedFuture(null);
+        }
+        return CompletableFuture.completedFuture(splitBoundaries);
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 57590d8d7da0e..ae7ceeed92838 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -133,6 +133,7 @@ public void setup() throws Exception {
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
         conf.setLoadBalancerSheddingEnabled(false);
+        conf.setLoadBalancerDebugModeEnabled(true);
         super.internalSetup(conf);
         pulsar1 = pulsar;
         ServiceConfiguration defaultConf = getDefaultConf();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 2d98bc5cae6e5..77c80187a63e1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -107,9 +107,16 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
     private String lookupServiceAddress1;
     private String lookupServiceAddress2;
     private String bundle;
-
     private String bundle1;
     private String bundle2;
+    private String bundle3;
+    private String childBundle1Range;
+    private String childBundle2Range;
+    private String childBundle11;
+    private String childBundle12;
+
+    private String childBundle31;
+    private String childBundle32;
     private PulsarTestContext additionalPulsarTestContext;
 
     private LoadManagerContext loadManagerContext;
@@ -122,6 +129,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
     @Override
     protected void setup() throws Exception {
         conf.setAllowAutoTopicCreation(true);
+        conf.setLoadBalancerDebugModeEnabled(true);
         conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10);
         super.internalSetup(conf);
 
@@ -149,15 +157,23 @@ protected void setup() throws Exception {
         lookupServiceAddress2 = (String)
                 FieldUtils.readDeclaredField(channel2, "lookupServiceAddress", true);
 
-        bundle = String.format("%s/%s", "public/default", "0x00000000_0xffffffff");
-        bundle1 = String.format("%s/%s", "public/default", "0x00000000_0xfffffff0");
-        bundle2 = String.format("%s/%s", "public/default", "0xfffffff0_0xffffffff");
+        bundle = "public/default/0x00000000_0xffffffff";
+        bundle1 = "public/default/0x00000000_0xfffffff0";
+        bundle2 = "public/default/0xfffffff0_0xffffffff";
+        bundle3 = "public/default3/0x00000000_0xffffffff";
+        childBundle1Range = "0x7fffffff_0xffffffff";
+        childBundle2Range = "0x00000000_0x7fffffff";
+
+        childBundle11 = "public/default/" + childBundle1Range;
+        childBundle12 = "public/default/" + childBundle2Range;
+
+        childBundle31 = "public/default3/" + childBundle1Range;
+        childBundle32 = "public/default3/" + childBundle2Range;
     }
 
     @BeforeMethod
     protected void initChannels() throws Exception {
-        cleanTableView(channel1, bundle);
-        cleanTableView(channel2, bundle);
+        cleanTableViews();
         cleanOwnershipMonitorCounters(channel1);
         cleanOwnershipMonitorCounters(channel2);
         cleanOpsCounters(channel1);
@@ -299,7 +315,9 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel)
             }
         }
         try {
-            channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1))
+            Split split = new Split(bundle, lookupServiceAddress1, Map.of(
+                    childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty()));
+            channel.publishSplitEventAsync(split)
                     .get(2, TimeUnit.SECONDS);
         } catch (ExecutionException e) {
             if (e.getCause() instanceof IllegalStateException) {
@@ -496,7 +514,8 @@ public void transferTestWhenDestBrokerFails()
         assertEquals(0, getOwnerRequests2.size());
 
         // recovered, check the monitor update state : Assigned -> Owned
-        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))).when(loadManager).selectAsync(any());
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1)))
+                .when(loadManager).selectAsync(any());
         FieldUtils.writeDeclaredField(channel2, "producer", producer, true);
         FieldUtils.writeDeclaredField(channel1,
                 "inFlightStateWaitingTimeInMillis", 1 , true);
@@ -564,32 +583,33 @@ public void splitAndRetryTest() throws Exception {
         doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
                 .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
 
-        Split split = new Split(bundle, ownerAddr1.get());
+        // Assert child bundle ownerships in the channels.
+
+        Split split = new Split(bundle, ownerAddr1.get(), Map.of(
+                childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty()));
         channel1.publishSplitEventAsync(split);
 
         waitUntilState(channel1, bundle, Deleted);
         waitUntilState(channel2, bundle, Deleted);
 
-        validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0);
-        validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0);
+        validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0);
+        validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0);
         validateEventCounters(channel1, 1, 0, 1, 0, 0, 0);
         validateEventCounters(channel2, 0, 0, 0, 0, 0, 0);
         // Verify the retry count
         verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1))
                 .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), any(), any(), anyLong(), any());
 
-        // Assert child bundle ownerships in the channels.
-        String childBundle1 = "public/default/0x7fffffff_0xffffffff";
-        String childBundle2 = "public/default/0x00000000_0x7fffffff";
 
-        waitUntilNewOwner(channel1, childBundle1, lookupServiceAddress1);
-        waitUntilNewOwner(channel1, childBundle2, lookupServiceAddress1);
-        waitUntilNewOwner(channel2, childBundle1, lookupServiceAddress1);
-        waitUntilNewOwner(channel2, childBundle2, lookupServiceAddress1);
-        assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle1).get());
-        assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle2).get());
-        assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle1).get());
-        assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle2).get());
+
+        waitUntilNewOwner(channel1, childBundle11, lookupServiceAddress1);
+        waitUntilNewOwner(channel1, childBundle12, lookupServiceAddress1);
+        waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress1);
+        waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress1);
+        assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle11).get());
+        assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle12).get());
+        assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle11).get());
+        assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle12).get());
 
 
         // try the monitor and check the monitor moves `Deleted` -> `Init`
@@ -620,10 +640,10 @@ public void splitAndRetryTest() throws Exception {
                 0,
                 0);
 
-        cleanTableView(channel1, childBundle1);
-        cleanTableView(channel2, childBundle1);
-        cleanTableView(channel1, childBundle2);
-        cleanTableView(channel2, childBundle2);
+        cleanTableView(channel1, childBundle11);
+        cleanTableView(channel2, childBundle11);
+        cleanTableView(channel1, childBundle12);
+        cleanTableView(channel2, childBundle12);
 
         FieldUtils.writeDeclaredField(channel1,
                 "inFlightStateWaitingTimeInMillis", 30 * 1000, true);
@@ -714,8 +734,8 @@ public void handleBrokerDeletionEventTest()
 
         var owner1 = channel1.getOwnerAsync(bundle1);
         var owner2 = channel2.getOwnerAsync(bundle2);
-        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any());
-
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
+                .when(loadManager).selectAsync(any());
         assertTrue(owner1.get().isEmpty());
         assertTrue(owner2.get().isEmpty());
 
@@ -916,8 +936,12 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx
         Awaitility.await()
                 .pollInterval(200, TimeUnit.MILLISECONDS)
                 .atMost(140, TimeUnit.SECONDS)
-                .untilAsserted(() -> verify(compactor, times(1))
-                        .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()));
+                .untilAsserted(() -> {
+                    channel1.publishAssignEventAsync(bundle, lookupServiceAddress1);
+                    verify(compactor, times(1))
+                            .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any());
+                });
+
 
         var channel3 = createChannel(pulsar);
         channel3.start();
@@ -1049,7 +1073,7 @@ public void unloadTest()
     }
 
     @Test(priority = 13)
-    public void assignTestWhenDestBrokerFails()
+    public void assignTestWhenDestBrokerProducerFails()
             throws ExecutionException, InterruptedException, IllegalAccessException {
 
         Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty());
@@ -1076,7 +1100,8 @@ public void assignTestWhenDestBrokerFails()
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
         FieldUtils.writeDeclaredField(channel2,
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
-        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))).when(loadManager).selectAsync(any());
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
+                .when(loadManager).selectAsync(any());
         channel1.publishAssignEventAsync(bundle, lookupServiceAddress2);
         // channel1 is broken. the assign won't be complete.
         waitUntilState(channel1, bundle);
@@ -1132,7 +1157,7 @@ public void assignTestWhenDestBrokerFails()
     }
 
     @Test(priority = 14)
-    public void splitTestWhenDestBrokerFails()
+    public void splitTestWhenProducerFails()
             throws ExecutionException, InterruptedException, IllegalAccessException {
 
 
@@ -1165,7 +1190,12 @@ public void splitTestWhenDestBrokerFails()
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
         FieldUtils.writeDeclaredField(channel2,
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
-        channel2.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, null));
+        // Assert child bundle ownerships in the channels.
+
+
+        Split split = new Split(bundle, lookupServiceAddress1, Map.of(
+                childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty()));
+        channel2.publishSplitEventAsync(split);
         // channel1 is broken. the split won't be complete.
         waitUntilState(channel1, bundle);
         waitUntilState(channel2, bundle);
@@ -1180,29 +1210,19 @@ public void splitTestWhenDestBrokerFails()
         FieldUtils.writeDeclaredField(channel2,
                 "inFlightStateWaitingTimeInMillis", 1 , true);
 
-        ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships(
-                List.of(lookupServiceAddress1, lookupServiceAddress2));
-        ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships(
-                List.of(lookupServiceAddress1, lookupServiceAddress2));
 
+        var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2;
 
-        waitUntilNewOwner(channel1, bundle, lookupServiceAddress1);
-        waitUntilNewOwner(channel2, bundle, lookupServiceAddress1);
-        var ownerAddr1 = channel1.getOwnerAsync(bundle).get();
-        var ownerAddr2 = channel2.getOwnerAsync(bundle).get();
+        waitUntilStateWithMonitor(leader, bundle, Deleted);
+        waitUntilStateWithMonitor(channel1, bundle, Deleted);
+        waitUntilStateWithMonitor(channel2, bundle, Deleted);
 
-        assertEquals(ownerAddr1, ownerAddr2);
-        assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1));
+        var ownerAddr1 = channel1.getOwnerAsync(bundle);
+        var ownerAddr2 = channel2.getOwnerAsync(bundle);
+
+        assertTrue(ownerAddr1.isCompletedExceptionally());
+        assertTrue(ownerAddr2.isCompletedExceptionally());
 
-        var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2;
-        validateMonitorCounters(leader,
-                0,
-                0,
-                1,
-                0,
-                0,
-                0,
-                0);
 
         FieldUtils.writeDeclaredField(channel1,
                 "inFlightStateWaitingTimeInMillis", 30 * 1000, true);
@@ -1273,6 +1293,243 @@ public void testIsOwner() throws IllegalAccessException {
         assertFalse(channel1.isOwner(bundle));
     }
 
+    @Test(priority = 16)
+    public void splitAndRetryFailureTest() throws Exception {
+        channel1.publishAssignEventAsync(bundle3, lookupServiceAddress1);
+        waitUntilNewOwner(channel1, bundle3, lookupServiceAddress1);
+        waitUntilNewOwner(channel2, bundle3, lookupServiceAddress1);
+        var ownerAddr1 = channel1.getOwnerAsync(bundle3).get();
+        var ownerAddr2 = channel2.getOwnerAsync(bundle3).get();
+        assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1));
+        assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1));
+        assertTrue(ownerAddr1.isPresent());
+
+        NamespaceService namespaceService = spy(pulsar1.getNamespaceService());
+        CompletableFuture future = new CompletableFuture<>();
+        int badVersionExceptionCount = 10;
+        AtomicInteger count = new AtomicInteger(badVersionExceptionCount);
+        future.completeExceptionally(new MetadataStoreException.BadVersionException("BadVersion"));
+        doAnswer(invocationOnMock -> {
+            if (count.decrementAndGet() > 0) {
+                return future;
+            }
+            // Call the real method
+            reset(namespaceService);
+            doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
+                    .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
+            return future;
+        }).when(namespaceService).updateNamespaceBundlesForPolicies(any(), any());
+        doReturn(namespaceService).when(pulsar1).getNamespaceService();
+        doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
+                .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
+
+        // Assert child bundle ownerships in the channels.
+
+        Split split = new Split(bundle3, ownerAddr1.get(), Map.of(
+                childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty()));
+        channel1.publishSplitEventAsync(split);
+
+        FieldUtils.writeDeclaredField(channel1,
+                "inFlightStateWaitingTimeInMillis", 1 , true);
+        FieldUtils.writeDeclaredField(channel2,
+                "inFlightStateWaitingTimeInMillis", 1 , true);
+
+        Awaitility.await()
+                .pollInterval(200, TimeUnit.MILLISECONDS)
+                .atMost(10, TimeUnit.SECONDS)
+                .untilAsserted(() -> {
+                    assertEquals(3, count.get());
+                });
+        var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2;
+        ((ServiceUnitStateChannelImpl) leader)
+                .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2));
+        waitUntilState(leader, bundle3, Deleted);
+        waitUntilState(channel1, bundle3, Deleted);
+        waitUntilState(channel2, bundle3, Deleted);
+
+
+        validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 0);
+        validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0);
+        validateEventCounters(channel1, 1, 0, 1, 0, 0, 0);
+        validateEventCounters(channel2, 0, 0, 0, 0, 0, 0);
+
+        waitUntilNewOwner(channel1, childBundle31, lookupServiceAddress1);
+        waitUntilNewOwner(channel1, childBundle32, lookupServiceAddress1);
+        waitUntilNewOwner(channel2, childBundle31, lookupServiceAddress1);
+        waitUntilNewOwner(channel2, childBundle32, lookupServiceAddress1);
+        assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle31).get());
+        assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle32).get());
+        assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle31).get());
+        assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle32).get());
+
+
+        // try the monitor and check the monitor moves `Deleted` -> `Init`
+
+        FieldUtils.writeDeclaredField(channel1,
+                "semiTerminalStateWaitingTimeInMillis", 1, true);
+        FieldUtils.writeDeclaredField(channel2,
+                "semiTerminalStateWaitingTimeInMillis", 1, true);
+
+        ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships(
+                List.of(lookupServiceAddress1, lookupServiceAddress2));
+        ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships(
+                List.of(lookupServiceAddress1, lookupServiceAddress2));
+        waitUntilState(channel1, bundle3, Init);
+        waitUntilState(channel2, bundle3, Init);
+
+        validateMonitorCounters(leader,
+                0,
+                1,
+                1,
+                0,
+                0,
+                0,
+                0);
+
+
+        cleanTableViews();
+
+        FieldUtils.writeDeclaredField(channel1,
+                "inFlightStateWaitingTimeInMillis", 30 * 1000, true);
+        FieldUtils.writeDeclaredField(channel1,
+                "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true);
+
+        FieldUtils.writeDeclaredField(channel2,
+                "inFlightStateWaitingTimeInMillis", 30 * 1000, true);
+        FieldUtils.writeDeclaredField(channel2,
+                "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true);
+    }
+
+    @Test(priority = 17)
+    public void testOverrideInactiveBrokerStateData()
+            throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException {
+
+        var leaderChannel = channel1;
+        var followerChannel = channel2;
+        String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get();
+        String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get();
+        assertEquals(leader, leader2);
+        if (leader.equals(lookupServiceAddress2)) {
+            leaderChannel = channel2;
+            followerChannel = channel1;
+        }
+
+        String broker = lookupServiceAddress1;
+
+        // test override states
+        String releasingBundle = "public/releasing/0xfffffff0_0xffffffff";
+        String splittingBundle = bundle;
+        String assigningBundle = "public/assigning/0xfffffff0_0xffffffff";
+        String freeBundle = "public/free/0xfffffff0_0xffffffff";
+        String deletedBundle = "public/deleted/0xfffffff0_0xffffffff";
+        String ownedBundle = "public/owned/0xfffffff0_0xffffffff";
+        overrideTableViews(releasingBundle,
+                new ServiceUnitStateData(Releasing, null, broker, 1));
+        overrideTableViews(splittingBundle,
+                new ServiceUnitStateData(Splitting, null, broker,
+                        Map.of(childBundle1Range, Optional.empty(),
+                                childBundle2Range, Optional.empty()), 1));
+        overrideTableViews(assigningBundle,
+                new ServiceUnitStateData(Assigning, broker, null, 1));
+        overrideTableViews(freeBundle,
+                new ServiceUnitStateData(Free, null, broker, 1));
+        overrideTableViews(deletedBundle,
+                new ServiceUnitStateData(Deleted, null, broker, 1));
+        overrideTableViews(ownedBundle,
+                new ServiceUnitStateData(Owned, broker, null, 1));
+
+        // test stable metadata state
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
+                .when(loadManager).selectAsync(any());
+        leaderChannel.handleMetadataSessionEvent(SessionReestablished);
+        followerChannel.handleMetadataSessionEvent(SessionReestablished);
+        FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp",
+                System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true);
+        FieldUtils.writeDeclaredField(followerChannel, "lastMetadataSessionEventTimestamp",
+                System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true);
+        leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted);
+        followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted);
+
+        waitUntilNewOwner(channel2, releasingBundle, lookupServiceAddress2);
+        waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress2);
+        waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress2);
+        waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2);
+        waitUntilNewOwner(channel2, ownedBundle, lookupServiceAddress2);
+        assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get());
+        assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally());
+        assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally());
+
+        // clean-up
+        FieldUtils.writeDeclaredField(leaderChannel, "maxCleanupDelayTimeInSecs", 3 * 60, true);
+        cleanTableViews();
+
+    }
+
+    @Test(priority = 18)
+    public void testOverrideOrphanStateData()
+            throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException {
+
+        var leaderChannel = channel1;
+        var followerChannel = channel2;
+        String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get();
+        String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get();
+        assertEquals(leader, leader2);
+        if (leader.equals(lookupServiceAddress2)) {
+            leaderChannel = channel2;
+            followerChannel = channel1;
+        }
+
+        String broker = lookupServiceAddress1;
+
+        // test override states
+        String releasingBundle = "public/releasing/0xfffffff0_0xffffffff";
+        String splittingBundle = bundle;
+        String assigningBundle = "public/assigning/0xfffffff0_0xffffffff";
+        String freeBundle = "public/free/0xfffffff0_0xffffffff";
+        String deletedBundle = "public/deleted/0xfffffff0_0xffffffff";
+        String ownedBundle = "public/owned/0xfffffff0_0xffffffff";
+        overrideTableViews(releasingBundle,
+                new ServiceUnitStateData(Releasing, null, broker, 1));
+        overrideTableViews(splittingBundle,
+                new ServiceUnitStateData(Splitting, null, broker,
+                        Map.of(childBundle1Range, Optional.empty(),
+                                childBundle2Range, Optional.empty()), 1));
+        overrideTableViews(assigningBundle,
+                new ServiceUnitStateData(Assigning, broker, null, 1));
+        overrideTableViews(freeBundle,
+                new ServiceUnitStateData(Free, null, broker, 1));
+        overrideTableViews(deletedBundle,
+                new ServiceUnitStateData(Deleted, null, broker, 1));
+        overrideTableViews(ownedBundle,
+                new ServiceUnitStateData(Owned, broker, null, 1));
+
+        // test stable metadata state
+        doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
+                .when(loadManager).selectAsync(any());
+        FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis",
+                -1, true);
+        FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis",
+                -1, true);
+        ((ServiceUnitStateChannelImpl) leaderChannel)
+                .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2));
+
+        waitUntilNewOwner(channel2, releasingBundle, broker);
+        waitUntilNewOwner(channel2, childBundle11, broker);
+        waitUntilNewOwner(channel2, childBundle12, broker);
+        waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2);
+        waitUntilNewOwner(channel2, ownedBundle, broker);
+        assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get());
+        assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally());
+        assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally());
+
+        // clean-up
+        FieldUtils.writeDeclaredField(channel1,
+                "inFlightStateWaitingTimeInMillis", 30 * 1000, true);
+        FieldUtils.writeDeclaredField(channel2,
+                "inFlightStateWaitingTimeInMillis", 30 * 1000, true);
+        cleanTableViews();
+    }
+
 
     private static ConcurrentOpenHashMap>> getOwnerRequests(
             ServiceUnitStateChannel channel) throws IllegalAccessException {
@@ -1375,6 +1632,22 @@ private static void waitUntilState(ServiceUnitStateChannel channel, String key,
                 });
     }
 
+    private void waitUntilStateWithMonitor(ServiceUnitStateChannel channel, String key, ServiceUnitState expected)
+            throws IllegalAccessException {
+        TableViewImpl tv = (TableViewImpl)
+                FieldUtils.readField(channel, "tableview", true);
+        Awaitility.await()
+                .pollInterval(200, TimeUnit.MILLISECONDS)
+                .atMost(10, TimeUnit.SECONDS)
+                .until(() -> { // wait until true
+                    ((ServiceUnitStateChannelImpl) channel)
+                            .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2));
+                    ServiceUnitStateData data = tv.get(key);
+                    ServiceUnitState actual = state(data);
+                    return actual == expected;
+                });
+    }
+
     private static void cleanTableView(ServiceUnitStateChannel channel, String serviceUnit)
             throws IllegalAccessException {
         var tv = (TableViewImpl)
@@ -1384,6 +1657,26 @@ private static void cleanTableView(ServiceUnitStateChannel channel, String servi
         cache.remove(serviceUnit);
     }
 
+    private void cleanTableViews()
+            throws IllegalAccessException {
+        var tv1 = (TableViewImpl)
+                FieldUtils.readField(channel1, "tableview", true);
+        var cache1 = (ConcurrentMap)
+                FieldUtils.readField(tv1, "data", true);
+        cache1.clear();
+
+        var tv2 = (TableViewImpl)
+                FieldUtils.readField(channel2, "tableview", true);
+        var cache2 = (ConcurrentMap)
+                FieldUtils.readField(tv2, "data", true);
+        cache2.clear();
+    }
+
+    private void overrideTableViews(String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException {
+        overrideTableView(channel1, serviceUnit, val);
+        overrideTableView(channel2, serviceUnit, val);
+    }
+
     private static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val)
             throws IllegalAccessException {
         var tv = (TableViewImpl)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
index 1e23ea74dca5e..fd50f0d8aedc2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.broker.loadbalance.extensions.channel.models;
 
 import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNull;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -56,12 +55,9 @@ public void testInvalidSplitServiceUnitToDestBroker() {
         new Split("A", "B", map);
     }
 
-    @Test
+    @Test(expectedExceptions = IllegalArgumentException.class)
     public void testNullSplitServiceUnitToDestBroker() {
-        var split = new Split("A", "B");
-        assertEquals(split.serviceUnit(), "A");
-        assertEquals(split.sourceBroker(), "B");
-        assertNull(split.splitServiceUnitToDestBroker());
+        var split = new Split("A", "B", null);
     }
 
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
index f7726f8e7a1aa..f56ad90833bdb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
@@ -28,6 +28,8 @@
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicReference;
@@ -56,6 +58,15 @@ public class SplitSchedulerTest {
     NamespaceBundleSplitStrategy strategy;
     String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF";
     String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF";
+
+    String childBundle12 = "tenant/namespace/0x7fffffff_0xffffffff";
+
+    String childBundle11 = "tenant/namespace/0x00000000_0x7fffffff";
+
+    String childBundle22 = "tenant/namespace/0x7fffffff_0x0fffffff";
+
+    String childBundle21 = "tenant/namespace/0x00000000_0x7fffffff";
+
     String broker = "broker-1";
     SplitDecision decision1;
     SplitDecision decision2;
@@ -77,11 +88,15 @@ public void setUp() {
         doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any());
 
         decision1 = new SplitDecision();
-        decision1.setSplit(new Split(bundle1, broker));
+        Split split = new Split(bundle1, broker, Map.of(
+                childBundle11, Optional.empty(), childBundle12, Optional.empty()));
+        decision1.setSplit(split);
         decision1.succeed(SplitDecision.Reason.MsgRate);
 
         decision2 = new SplitDecision();
-        decision2.setSplit(new Split(bundle2, broker));
+        Split split2 = new Split(bundle2, broker, Map.of(
+                childBundle21, Optional.empty(), childBundle22, Optional.empty()));
+        decision2.setSplit(split2);
         decision2.succeed(SplitDecision.Reason.Sessions);
         Set decisions = Set.of(decision1, decision2);
         doReturn(decisions).when(strategy).findBundlesToSplit(any(), any());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
index 8c765e7a3df64..8b489c92af0af 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -28,9 +28,13 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
+import com.google.common.hash.Hashing;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -38,15 +42,19 @@
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
-import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -71,7 +79,12 @@ public class DefaultNamespaceBundleSplitStrategyTest {
 
     String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF";
     String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF";
-
+    Long splitBoundary1 = 0x7fffffffL;
+    Long splitBoundary2 = 0x07ffffffL;
+    String childBundle12 = "0x7fffffff_0xffffffff";
+    String childBundle11 = "0x00000000_0x7fffffff";
+    String childBundle22 = "0x07ffffff_0x0fffffff";
+    String childBundle21 = "0x00000000_0x07ffffff";
     String broker = "broker-1";
 
     @BeforeMethod
@@ -90,19 +103,21 @@ void setup() {
         brokerService = mock(BrokerService.class);
         pulsarStats = mock(PulsarStats.class);
         namespaceService = mock(NamespaceService.class);
-        namespaceBundleFactory = mock(NamespaceBundleFactory.class);
+
         loadManagerContext = mock(LoadManagerContext.class);
         brokerRegistry = mock(BrokerRegistry.class);
         loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class);
         loadManager = mock(ExtensibleLoadManagerImpl.class);
         channel = mock(ServiceUnitStateChannelImpl.class);
 
+
+        doReturn(mock(MetadataStoreExtended.class)).when(pulsar).getLocalMetadataStore();
+        namespaceBundleFactory = spy(new NamespaceBundleFactory(pulsar, Hashing.crc32()));
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(namespaceService).when(pulsar).getNamespaceService();
         doReturn(namespaceBundleFactory).when(namespaceService).getNamespaceBundleFactory();
-        doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
         doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry();
         doReturn(broker).when(brokerRegistry).getBrokerId();
         doReturn(new AtomicReference(loadManagerWrapper)).when(pulsar).getLoadManager();
@@ -110,6 +125,18 @@ void setup() {
         doReturn(channel).when(loadManager).getServiceUnitStateChannel();
         doReturn(true).when(channel).isOwner(any());
 
+        var namespaceBundle1 = namespaceBundleFactory.getBundle(
+                LoadManagerShared.getNamespaceNameFromBundleName(bundle1),
+                LoadManagerShared.getBundleRangeFromBundleName(bundle1));
+        var namespaceBundle2 = namespaceBundleFactory.getBundle(
+                LoadManagerShared.getNamespaceNameFromBundleName(bundle2),
+                LoadManagerShared.getBundleRangeFromBundleName(bundle2));
+        doReturn(CompletableFuture.completedFuture(
+                List.of(splitBoundary1))).when(namespaceService).getSplitBoundary(
+                        eq(namespaceBundle1), eq((List)null), any());
+        doReturn(CompletableFuture.completedFuture(
+                List.of(splitBoundary2))).when(namespaceService).getSplitBoundary(
+                        eq(namespaceBundle2), eq((List)null), any());
 
         bundleStats = new LinkedHashMap<>();
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
@@ -184,6 +211,20 @@ public void testError() throws Exception {
         verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
     }
 
+    public void testSplittingBundle() {
+        var counter = spy(new SplitCounter());
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        doReturn(Map.of("tenant/namespace/0x00000000_0xFFFFFFFF",
+                new ServiceUnitStateData(ServiceUnitState.Splitting, broker, 1)).entrySet())
+                .when(channel).getOwnershipEntrySet();
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+        verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+    }
+
     public void testMaxMsgRate() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
@@ -196,14 +237,18 @@ public void testMaxMsgRate() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker));
+                Split split = new Split(bundle1, broker, Map.of(
+                        childBundle11, Optional.empty(), childBundle12, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.MsgRate);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker));
+                Split split = new Split(bundle2, broker, Map.of(
+                        childBundle21, Optional.empty(), childBundle22, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.MsgRate);
 
                 assertEquals(actual, Set.of(decision1));
@@ -224,14 +269,18 @@ public void testMaxTopics() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker));
+                Split split = new Split(bundle1, broker, Map.of(
+                        childBundle11, Optional.empty(), childBundle12, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.Topics);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker));
+                Split split = new Split(bundle2, broker, Map.of(
+                        childBundle21, Optional.empty(), childBundle22, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.Topics);
 
                 assertEquals(actual, Set.of(decision1));
@@ -255,14 +304,18 @@ public void testMaxSessions() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker));
+                Split split = new Split(bundle1, broker, Map.of(
+                        childBundle11, Optional.empty(), childBundle12, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.Sessions);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker));
+                Split split = new Split(bundle2, broker, Map.of(
+                        childBundle21, Optional.empty(), childBundle22, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.Sessions);
 
                 assertEquals(actual, Set.of(decision1));
@@ -286,14 +339,18 @@ public void testMaxBandwidthMbytes() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker));
+                Split split = new Split(bundle1, broker, Map.of(
+                        childBundle11, Optional.empty(), childBundle12, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.Bandwidth);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker));
+                Split split = new Split(bundle2, broker, Map.of(
+                        childBundle21, Optional.empty(), childBundle22, Optional.empty()));
+                decision1.setSplit(split);
                 decision1.succeed(SplitDecision.Reason.Bandwidth);
 
                 assertEquals(actual, Set.of(decision1));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java
index ec458def65bb7..80f9c279a55b7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithmTest.java
@@ -34,6 +34,21 @@
 
 public class SpecifiedPositionsBundleSplitAlgorithmTest {
 
+    @Test
+    public void testEmptyTopicWithForce() {
+        SpecifiedPositionsBundleSplitAlgorithm algorithm = new SpecifiedPositionsBundleSplitAlgorithm(true);
+        NamespaceService mockNamespaceService = mock(NamespaceService.class);
+        NamespaceBundle mockNamespaceBundle = mock(NamespaceBundle.class);
+        doReturn(1L).when(mockNamespaceBundle).getLowerEndpoint();
+        doReturn(1000L).when(mockNamespaceBundle).getUpperEndpoint();
+        doReturn(CompletableFuture.completedFuture(List.of()))
+                .when(mockNamespaceService).getOwnedTopicListForNamespaceBundle(mockNamespaceBundle);
+        List splitPositions =
+                algorithm.getSplitBoundary(new BundleSplitOption(mockNamespaceService, mockNamespaceBundle,
+                        Arrays.asList(1L, 2L))).join();
+        assertEquals(splitPositions, Arrays.asList(2L));
+    }
+
     @Test
     public void testTotalTopicsSizeLessThan1() {
         SpecifiedPositionsBundleSplitAlgorithm algorithm = new SpecifiedPositionsBundleSplitAlgorithm();

From 90b9dd4c432ce14a28f76baf2e5a1bbd90b44b89 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Sat, 8 Apr 2023 15:41:43 -0500
Subject: [PATCH 154/404] [fix][fn] Make /version return correct version
 (#20047)

### Motivation

The `/version` endpoint for functions currently returns `{"version":"version.number"}` instead of an actual version number. This PR fixes that.

One observation is that the broker's `/version` endpoint returns a plain string:

https://github.com/apache/pulsar/blob/82237d3684fe506bcb6426b3b23f413422e6e4fb/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java#L527-L535

### Modifications

* Return the actual version.
* Use a static field to decrease the cost of this endpoint.

### Verifying this change

This is a trivial change.

### Documentation

- [x] `doc-not-needed`

This fixes an endpoint, no docs are needed.

### Matching PR in forked repository

PR in forked repository: Skipping PR since this is so trivial
---
 .../worker/rest/ConfigurationResource.java        | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java
index a801d8503d3ee..2218431770b66 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/ConfigurationResource.java
@@ -19,30 +19,25 @@
 package org.apache.pulsar.functions.worker.rest;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.PulsarVersion;
 
 @Path("/")
 public class ConfigurationResource {
+
+    private static final String VERSION = "{\"version\":\"" + PulsarVersion.getVersion() + "\"}";
+
     @Path("version")
     @GET
     @Produces(MediaType.APPLICATION_JSON)
     public Response release() throws JsonProcessingException {
-        final ObjectMapper mapper = ObjectMapperFactory.getMapper().getObjectMapper();
-        final ObjectNode node = mapper.createObjectNode();
-        node.put("version", "version.number");
-
         return Response.ok()
                 .type(MediaType.APPLICATION_JSON)
-                .entity(mapper
-                        .writerWithDefaultPrettyPrinter()
-                        .writeValueAsString(node))
+                .entity(VERSION)
                 .build();
     }
 }

From 470b674016c8718f2dfd0a0f93cf02d49af0fead Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 10 Apr 2023 15:06:33 +0800
Subject: [PATCH 155/404] [feat][sql] Bump Trino version to 368 and fix Decimal
 breaking change (#20016)

Signed-off-by: tison 
---
 pom.xml                                       |  2 +-
 pulsar-sql/pom.xml                            |  2 +-
 pulsar-sql/presto-distribution/LICENSE        | 75 ++++++++++---------
 pulsar-sql/presto-distribution/pom.xml        |  4 +
 pulsar-sql/presto-pulsar/pom.xml              |  7 +-
 .../decoder/avro/PulsarAvroColumnDecoder.java | 39 +++++++---
 .../decoder/json/PulsarJsonFieldDecoder.java  | 49 ++++++++----
 .../json/PulsarJsonRowDecoderFactory.java     | 14 ++--
 .../sql/presto/decoder/DecoderTestUtil.java   | 19 +++--
 .../presto/decoder/avro/TestAvroDecoder.java  | 36 +++++----
 10 files changed, 145 insertions(+), 102 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4534036f3fd2c..7f20d478594ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -184,7 +184,7 @@ flexible messaging model and an intuitive client API.
     2.4.10
     1.2.4
     8.5.2
-    363
+    368
     1.9.7.Final
     42.5.0
     8.0.30
diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml
index 4dbf7d5737335..1f8972bf22219 100644
--- a/pulsar-sql/pom.xml
+++ b/pulsar-sql/pom.xml
@@ -36,7 +36,7 @@
         3.14.9
         
         1.17.2
-        208
+        213
     
 
     
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index c523fae7606e1..6d60765d459b1 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -303,28 +303,28 @@ The Apache Software License, Version 2.0
     - bytecode-1.2.jar
   * Airlift
     - aircompressor-0.20.jar
-    - bootstrap-208.jar
-    - concurrent-208.jar
-    - configuration-208.jar
-    - discovery-208.jar
+    - bootstrap-213.jar
+    - concurrent-213.jar
+    - configuration-213.jar
+    - discovery-213.jar
     - discovery-server-1.30.jar
-    - event-208.jar
-    - event-http-208.jar
-    - http-client-208.jar
-    - http-server-208.jar
-    - jmx-208.jar
-    - jmx-http-208.jar
-    - jmx-http-rpc-208.jar
+    - event-213.jar
+    - event-http-213.jar
+    - http-client-213.jar
+    - http-server-213.jar
+    - jmx-213.jar
+    - jmx-http-213.jar
+    - jmx-http-rpc-213.jar
     - joni-2.1.5.3.jar
-    - json-208.jar
-    - log-208.jar
-    - log-manager-208.jar
-    - node-208.jar
+    - json-213.jar
+    - log-213.jar
+    - log-manager-213.jar
+    - node-213.jar
     - parameternames-1.4.jar
-    - security-208.jar
-    - slice-0.39.jar
-    - stats-208.jar
-    - trace-token-208.jar
+    - security-213.jar
+    - slice-0.41.jar
+    - stats-213.jar
+    - trace-token-213.jar
     - units-1.6.jar
    * Apache HTTP Client
     - httpclient-4.5.13.jar
@@ -340,7 +340,9 @@ The Apache Software License, Version 2.0
   * J2ObjC Annotations
     - j2objc-annotations-1.3.jar
   * JSON Web Token Support For The JVM
-    - jjwt-0.9.0.jar
+    - jjwt-api-0.11.1.jar
+    - jjwt-impl-0.11.1.jar
+    - jjwt-jackson-0.11.1.jar
   * Jmxutils
     - jmxutils-1.21.jar
   * LevelDB
@@ -384,18 +386,18 @@ The Apache Software License, Version 2.0
   * Okio
     - okio-1.17.2.jar
   * Trino
-    - trino-array-363.jar
-    - trino-cli-363.jar
-    - trino-client-363.jar
-    - trino-geospatial-toolkit-363.jar
-    - trino-main-363.jar
-    - trino-matching-363.jar
-    - trino-memory-context-363.jar
-    - trino-parser-363.jar
-    - trino-plugin-toolkit-363.jar
-    - trino-server-main-363.jar
-    - trino-spi-363.jar
-    - trino-record-decoder-363.jar
+    - trino-array-368.jar
+    - trino-cli-368.jar
+    - trino-client-368.jar
+    - trino-geospatial-toolkit-368.jar
+    - trino-main-368.jar
+    - trino-matching-368.jar
+    - trino-memory-context-368.jar
+    - trino-parser-368.jar
+    - trino-plugin-toolkit-368.jar
+    - trino-server-main-368.jar
+    - trino-spi-368.jar
+    - trino-record-decoder-368.jar
   * RocksDB JNI
     - rocksdbjni-6.29.4.1.jar
   * SnakeYAML
@@ -456,6 +458,7 @@ The Apache Software License, Version 2.0
     - javassist-3.25.0-GA.jar
   * Java Native Access
     - jna-5.12.1.jar
+    - jna-platform-5.10.0.jar
   * Java Object Layout: Core
     - jol-core-0.2.jar
   * Yahoo Datasketches
@@ -493,7 +496,7 @@ BSD License
  * ANTLR 4 Runtime
     - antlr4-runtime-4.9.2.jar
  * ASM, a very small and fast Java bytecode manipulation framework
-    - asm-6.2.1.jar
+    - asm-9.1.jar
     - asm-analysis-6.2.1.jar
     - asm-tree-6.2.1.jar
     - asm-util-6.2.1.jar
@@ -518,6 +521,8 @@ MIT License
  * ScribeJava
    - scribejava-apis-6.9.0.jar
    - scribejava-core-6.9.0.jar
+ * OSHI
+   - oshi-core-5.8.5.jar
 
 CDDL - 1.0
  * OSGi Resource Locator
@@ -533,8 +538,9 @@ CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt
    - hk2-utils-2.6.1.jar
    - aopalliance-repackaged-2.6.1.jar
  * Jersey
-    - jaxrs-208.jar
+    - jaxrs-213.jar
     - jersey-client-2.34.jar
+    - jersey-common-2.34.jar
     - jersey-container-servlet-2.34.jar
     - jersey-container-servlet-core-2.34.jar
     - jersey-entity-filtering-2.34.jar
@@ -542,7 +548,6 @@ CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt
     - jersey-media-json-jackson-2.34.jar
     - jersey-media-multipart-2.34.jar
     - jersey-server-2.34.jar
-    - jersey-common-2.34.jar
  * JAXB
     - jaxb-api-2.3.1.jar
     - jaxb-runtime-2.3.4.jar
diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml
index 6e39bd50ad8be..48a1bf90662f6 100644
--- a/pulsar-sql/presto-distribution/pom.xml
+++ b/pulsar-sql/presto-distribution/pom.xml
@@ -97,6 +97,10 @@
           com.google.inject.extensions
           guice-multibindings
         
+        
+          org.apache.logging.log4j
+          log4j-to-slf4j
+        
       
     
 
diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml
index 622c66e6d76ba..e61580f1fcf15 100644
--- a/pulsar-sql/presto-pulsar/pom.xml
+++ b/pulsar-sql/presto-pulsar/pom.xml
@@ -41,8 +41,13 @@
         
             io.airlift
             bootstrap
+            
+                
+                    org.apache.logging.log4j
+                    log4j-to-slf4j
+                
+            
         
-
         
             io.airlift
             json
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java
index 4616a60379971..73081f8948a51 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java
@@ -41,8 +41,8 @@
 import io.trino.spi.type.BooleanType;
 import io.trino.spi.type.DateType;
 import io.trino.spi.type.DecimalType;
-import io.trino.spi.type.Decimals;
 import io.trino.spi.type.DoubleType;
+import io.trino.spi.type.Int128;
 import io.trino.spi.type.IntegerType;
 import io.trino.spi.type.MapType;
 import io.trino.spi.type.RealType;
@@ -66,11 +66,14 @@
 import org.apache.avro.generic.GenericRecord;
 
 /**
- * Copy from {@link io.trino.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345)
- * with A little bit pulsar's extensions.
- * 1) support {@link io.trino.spi.type.TimestampType},{@link io.trino.spi.type.DateType}DATE,
- *  * {@link io.trino.spi.type.TimeType}.
+ * Copy from {@link io.trino.decoder.avro.AvroColumnDecoder}
+ * with A little pulsar's extensions.
+ * 1) support date and time types.
+ *  {@link io.trino.spi.type.TimestampType}
+ *  {@link io.trino.spi.type.DateType}
+ *  {@link io.trino.spi.type.TimeType}
  * 2) support {@link io.trino.spi.type.RealType}.
+ * 3) support {@link io.trino.spi.type.DecimalType}.
  */
 public class PulsarAvroColumnDecoder {
     private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of(
@@ -252,13 +255,6 @@ private static Slice getSlice(Object value, Type type, String columnName) {
             }
         }
 
-        // The returned Slice size must be equals to 18 Byte
-        if (type instanceof DecimalType) {
-            ByteBuffer buffer = (ByteBuffer) value;
-            BigInteger bigInteger = new BigInteger(buffer.array());
-            return Decimals.encodeUnscaledValue(bigInteger);
-        }
-
         throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED,
                 format("cannot decode object of '%s' as '%s' for column '%s'",
                         value.getClass(), type, columnName));
@@ -274,6 +270,9 @@ private static Block serializeObject(BlockBuilder builder, Object value, Type ty
         if (type instanceof RowType) {
             return serializeRow(builder, value, type, columnName);
         }
+        if (type instanceof DecimalType && !((DecimalType) type).isShort()) {
+            return serializeLongDecimal(builder, value, type, columnName);
+        }
         serializePrimitive(builder, value, type, columnName);
         return null;
     }
@@ -299,6 +298,22 @@ private static Block serializeList(BlockBuilder parentBlockBuilder, Object value
         return blockBuilder.build();
     }
 
+    private static Block serializeLongDecimal(
+            BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) {
+        final BlockBuilder blockBuilder;
+        if (parentBlockBuilder != null) {
+            blockBuilder = parentBlockBuilder;
+        } else {
+            blockBuilder = type.createBlockBuilder(null, 1);
+        }
+        final ByteBuffer buffer = (ByteBuffer) value;
+        type.writeObject(blockBuilder, Int128.fromBigEndian(buffer.array()));
+        if (parentBlockBuilder == null) {
+            return blockBuilder.getSingleValueBlock(0);
+        }
+        return null;
+    }
+
     private static void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) {
         requireNonNull(blockBuilder, "parent blockBuilder is null");
 
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java
index c09dd43e24143..905e3bd6becb4 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java
@@ -29,6 +29,7 @@
 import static java.util.Objects.requireNonNull;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.DecimalNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableList;
 import io.airlift.log.Logger;
@@ -45,8 +46,8 @@
 import io.trino.spi.type.BooleanType;
 import io.trino.spi.type.DateType;
 import io.trino.spi.type.DecimalType;
-import io.trino.spi.type.Decimals;
 import io.trino.spi.type.DoubleType;
+import io.trino.spi.type.Int128;
 import io.trino.spi.type.IntegerType;
 import io.trino.spi.type.MapType;
 import io.trino.spi.type.RealType;
@@ -59,21 +60,22 @@
 import io.trino.spi.type.Type;
 import io.trino.spi.type.VarbinaryType;
 import io.trino.spi.type.VarcharType;
-import java.math.BigInteger;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.lang3.tuple.Pair;
 
 /**
- * Copy from {@link io.trino.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345)
- * with some pulsar's extensions.
+ * Copy from {@link io.trino.decoder.json.DefaultJsonFieldDecoder} with some pulsar's extensions.
  * 1) support {@link io.trino.spi.type.ArrayType}.
  * 2) support {@link io.trino.spi.type.MapType}.
  * 3) support {@link io.trino.spi.type.RowType}.
- * 4) support {@link io.trino.spi.type.TimestampType},{@link io.trino.spi.type.DateType},
- * {@link io.trino.spi.type.TimeType}.
+ * 4) support date and time types.
+ *  {@link io.trino.spi.type.TimestampType}
+ *  {@link io.trino.spi.type.DateType}
+ *  {@link io.trino.spi.type.TimeType}
  * 5) support {@link io.trino.spi.type.RealType}.
+ * 6) support {@link io.trino.spi.type.DecimalType}.
  */
 public class PulsarJsonFieldDecoder
         implements JsonFieldDecoder {
@@ -90,7 +92,6 @@ public PulsarJsonFieldDecoder(DecoderColumnHandle columnHandle) {
         Pair range = getNumRangeByType(columnHandle.getType());
         minValue = range.getKey();
         maxValue = range.getValue();
-
     }
 
     private static Pair getNumRangeByType(Type type) {
@@ -221,7 +222,7 @@ public static long getLong(JsonNode value, Type type, String columnName, long mi
                 }
 
                 // If it is decimalType, need to eliminate the decimal point,
-                // and give it to presto to set the decimal point
+                // and give it to trino to set the decimal point
                 if (type instanceof DecimalType) {
                     String decimalLong = value.asText().replace(".", "");
                     return Long.parseLong(decimalLong);
@@ -273,14 +274,6 @@ public static double getDouble(JsonNode value, Type type, String columnName) {
         private static Slice getSlice(JsonNode value, Type type, String columnName) {
             String textValue = value.isValueNode() ? value.asText() : value.toString();
 
-            // If it is decimalType, need to eliminate the decimal point,
-            // and give it to presto to set the decimal point
-            if (type instanceof DecimalType) {
-                textValue = textValue.replace(".", "");
-                BigInteger bigInteger = new BigInteger(textValue);
-                return Decimals.encodeUnscaledValue(bigInteger);
-            }
-
             Slice slice = utf8Slice(textValue);
             if (type instanceof VarcharType) {
                 slice = truncateToLength(slice, type);
@@ -298,6 +291,9 @@ private Block serializeObject(BlockBuilder builder, Object value, Type type, Str
             if (type instanceof RowType) {
                 return serializeRow(builder, value, type, columnName);
             }
+            if (type instanceof DecimalType && !((DecimalType) type).isShort()) {
+                return serializeLongDecimal(builder, value, type, columnName);
+            }
             serializePrimitive(builder, value, type, columnName);
             return null;
         }
@@ -330,6 +326,27 @@ private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type
             return blockBuilder.build();
         }
 
+        private static Block serializeLongDecimal(
+                BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) {
+            final BlockBuilder blockBuilder;
+            if (parentBlockBuilder != null) {
+                blockBuilder = parentBlockBuilder;
+            } else {
+                blockBuilder = type.createBlockBuilder(null, 1);
+            }
+
+            assert value instanceof DecimalNode;
+            final DecimalNode node = (DecimalNode) value;
+            // For decimalType, need to eliminate the decimal point,
+            // and give it to trino to set the decimal point
+            type.writeObject(blockBuilder, Int128.valueOf(node.asText().replace(".", "")));
+
+            if (parentBlockBuilder == null) {
+                return blockBuilder.getSingleValueBlock(0);
+            }
+            return null;
+        }
+
         private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type type, String columnName) {
             requireNonNull(blockBuilder, "parent blockBuilder is null");
 
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java
index 5dd06fa939621..0d5cc2d262dfe 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java
@@ -116,7 +116,7 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf
     }
 
 
-    private Type parseJsonPrestoType(String fieldname, Schema schema) {
+    private Type parseJsonPrestoType(String fieldName, Schema schema) {
         Schema.Type type = schema.getType();
         LogicalType logicalType  = schema.getLogicalType();
         switch (type) {
@@ -126,7 +126,7 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) {
             case NULL:
                 throw new UnsupportedOperationException(format(
                         "field '%s' NULL type code should not be reached , "
-                                + "please check the schema or report the bug.", fieldname));
+                                + "please check the schema or report the bug.", fieldName));
             case FIXED:
             case BYTES:
                 //  When the precision <= 0, throw Exception.
@@ -157,10 +157,10 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) {
             case BOOLEAN:
                 return BooleanType.BOOLEAN;
             case ARRAY:
-                return new ArrayType(parseJsonPrestoType(fieldname, schema.getElementType()));
+                return new ArrayType(parseJsonPrestoType(fieldName, schema.getElementType()));
             case MAP:
                 //The key for an avro map must be string.
-                TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature();
+                TypeSignature valueType = parseJsonPrestoType(fieldName, schema.getValueType()).getTypeSignature();
                 return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.
                         typeParameter(VarcharType.VARCHAR.getTypeSignature()),
                         TypeSignatureParameter.typeParameter(valueType)));
@@ -173,16 +173,16 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) {
                 } else {
                     throw new UnsupportedOperationException(format(
                             "field '%s' of record type has no fields, "
-                                    + "please check schema definition. ", fieldname));
+                                    + "please check schema definition. ", fieldName));
                 }
             case UNION:
                 for (Schema nestType : schema.getTypes()) {
                     if (nestType.getType() != Schema.Type.NULL) {
-                        return parseJsonPrestoType(fieldname, nestType);
+                        return parseJsonPrestoType(fieldName, nestType);
                     }
                 }
                 throw new UnsupportedOperationException(format(
-                        "field '%s' of UNION type must contains not NULL type.", fieldname));
+                        "field '%s' of UNION type must contains not NULL type.", fieldName));
             default:
                 throw new UnsupportedOperationException(format(
                         "Can't convert from schema type '%s' (%s) to presto type.",
diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java
index 84d93e780346a..60d6028f239d7 100644
--- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java
+++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java
@@ -18,24 +18,23 @@
  */
 package org.apache.pulsar.sql.presto.decoder;
 
+import static io.trino.testing.TestingConnectorSession.SESSION;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
 import io.airlift.slice.Slice;
 import io.trino.decoder.DecoderColumnHandle;
 import io.trino.decoder.FieldValueProvider;
 import io.trino.spi.block.Block;
 import io.trino.spi.type.ArrayType;
 import io.trino.spi.type.DecimalType;
-import io.trino.spi.type.Decimals;
+import io.trino.spi.type.Int128;
 import io.trino.spi.type.MapType;
 import io.trino.spi.type.RowType;
 import io.trino.spi.type.Type;
 import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.util.Map;
 
-import static io.trino.spi.type.UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH;
-import static io.trino.testing.TestingConnectorSession.SESSION;
-import static org.testng.Assert.*;
-
 /**
  * Abstract util superclass for  XXDecoderTestUtil (e.g. AvroDecoderTestUtil 、JsonDecoderTestUtil)
  */
@@ -122,10 +121,10 @@ public void checkValue(Map decodedRow,
         FieldValueProvider provider = decodedRow.get(handle);
         DecimalType decimalType = (DecimalType) handle.getType();
         BigDecimal actualDecimal;
-        if (decimalType.getFixedSize() == UNSCALED_DECIMAL_128_SLICE_LENGTH) {
-            Slice slice = provider.getSlice();
-            BigInteger bigInteger = Decimals.decodeUnscaledValue(slice);
-            actualDecimal = new BigDecimal(bigInteger, decimalType.getScale());
+        if (decimalType.getFixedSize() == Int128.SIZE) {
+            final Block block = provider.getBlock();
+            final Int128 actualValue = (Int128) decimalType.getObject(block, 0);
+            actualDecimal = new BigDecimal(actualValue.toBigInteger(), decimalType.getScale());
         } else {
             actualDecimal = BigDecimal.valueOf(provider.getLong(), decimalType.getScale());
         }
diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java
index 79ca0f5a65075..c4e7009b9465b 100644
--- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java
+++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java
@@ -18,6 +18,19 @@
  */
 package org.apache.pulsar.sql.presto.decoder.avro;
 
+import static io.trino.spi.type.BigintType.BIGINT;
+import static io.trino.spi.type.BooleanType.BOOLEAN;
+import static io.trino.spi.type.DoubleType.DOUBLE;
+import static io.trino.spi.type.IntegerType.INTEGER;
+import static io.trino.spi.type.RealType.REAL;
+import static io.trino.spi.type.TimeType.TIME_MILLIS;
+import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS;
+import static io.trino.spi.type.VarcharType.VARCHAR;
+import static java.lang.Float.floatToIntBits;
+import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.expectThrows;
 import com.google.common.collect.ImmutableList;
 import io.netty.buffer.ByteBuf;
 import io.trino.decoder.DecoderColumnHandle;
@@ -33,6 +46,10 @@
 import io.trino.spi.type.TypeSignatureParameter;
 import io.trino.spi.type.VarcharType;
 import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -47,25 +64,6 @@
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import java.time.LocalDate;
-import java.time.LocalTime;
-import java.time.ZoneId;
-import java.time.temporal.ChronoUnit;
-
-import static io.trino.spi.type.BigintType.BIGINT;
-import static io.trino.spi.type.BooleanType.BOOLEAN;
-import static io.trino.spi.type.DoubleType.DOUBLE;
-import static io.trino.spi.type.IntegerType.INTEGER;
-import static io.trino.spi.type.RealType.REAL;
-import static io.trino.spi.type.TimeType.TIME_MILLIS;
-import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS;
-import static io.trino.spi.type.VarcharType.VARCHAR;
-import static java.lang.Float.floatToIntBits;
-import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.expectThrows;
-
 public class TestAvroDecoder extends AbstractDecoderTester {
 
     private AvroSchema schema;

From 21c7c628d2a3fd7277f16fe7ba72154cfbf08128 Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Mon, 10 Apr 2023 22:34:27 +0800
Subject: [PATCH 156/404] PIP-254: Support configuring client version (#20009)

---
 .../client/api/MutualAuthenticationTest.java  | 48 ++++++++++++++++---
 .../api/SimpleProducerConsumerTest.java       | 34 ++++++++++++-
 .../pulsar/client/impl/ClientBuilderImpl.java | 23 +++++++++
 .../apache/pulsar/client/impl/ClientCnx.java  |  9 ++--
 .../impl/conf/ClientConfigurationData.java    |  6 +++
 5 files changed, 110 insertions(+), 10 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java
index 472af3e88cd36..2fc8aebf64a4a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java
@@ -26,20 +26,26 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 import javax.naming.AuthenticationException;
 import javax.net.ssl.SSLSession;
+import org.apache.pulsar.PulsarVersion;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
 import org.apache.pulsar.broker.authentication.AuthenticationProvider;
 import org.apache.pulsar.broker.authentication.AuthenticationState;
+import org.apache.pulsar.client.impl.ClientBuilderImpl;
 import org.apache.pulsar.common.api.AuthData;
+import org.apache.pulsar.common.policies.data.PublisherStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.testng.Assert.assertEquals;
 
 /**
  * Test Mutual Authentication.
@@ -182,7 +188,7 @@ public AuthenticationState newAuthState(AuthData authData,
         }
     }
 
-    @BeforeMethod(alwaysRun = true)
+    @BeforeClass(alwaysRun = true)
     @Override
     protected void setup() throws Exception {
         mutualAuth = new MutualAuthentication();
@@ -205,7 +211,7 @@ protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) {
         clientBuilder.authentication(mutualAuth);
     }
 
-    @AfterMethod(alwaysRun = true)
+    @AfterClass(alwaysRun = true)
     @Override
     protected void cleanup() throws Exception {
         internalCleanup();
@@ -214,12 +220,13 @@ protected void cleanup() throws Exception {
     @Test
     public void testAuthentication() throws Exception {
         log.info("-- Starting {} test --", methodName);
+        String topic = "persistent://my-property/my-ns/test-authentication";
 
-        Consumer consumer = pulsarClient.newConsumer().topic("persistent://my-property/my-ns/my-topic1")
+        Consumer consumer = pulsarClient.newConsumer().topic(topic)
             .subscriptionName("my-subscriber-name")
             .subscribe();
         Producer producer = pulsarClient.newProducer(Schema.BYTES)
-            .topic("persistent://my-property/my-ns/my-topic1")
+            .topic(topic)
             .create();
 
         for (int i = 0; i < 10; i++) {
@@ -239,4 +246,33 @@ public void testAuthentication() throws Exception {
 
         log.info("-- Exiting {} test --", methodName);
     }
+
+    @Test
+    public void testClientVersion() throws Exception {
+        String defaultClientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion();
+        String topic = "persistent://my-property/my-ns/test-client-version";
+
+        Producer producer1 = pulsarClient.newProducer()
+                .topic(topic)
+                .create();
+        TopicStats stats = admin.topics().getStats(topic);
+        assertEquals(stats.getPublishers().size(), 1);
+        assertEquals(stats.getPublishers().get(0).getClientVersion(), defaultClientVersion);
+
+        PulsarClient client = ((ClientBuilderImpl) PulsarClient.builder())
+                .description("my-java-client")
+                .serviceUrl(lookupUrl.toString())
+                .authentication(mutualAuth)
+                .build();
+        Producer producer2 = client.newProducer().topic(topic).create();
+        stats = admin.topics().getStats(topic);
+        assertEquals(stats.getPublishers().size(), 2);
+
+        assertEquals(stats.getPublishers().stream().map(PublisherStats::getClientVersion).collect(Collectors.toSet()),
+                Sets.newHashSet(defaultClientVersion, defaultClientVersion + "-my-java-client"));
+
+        producer1.close();
+        producer2.close();
+        client.close();
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 293e298fc6671..1bc437195d96d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -83,10 +83,12 @@
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.PulsarVersion;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.schema.GenericRecord;
+import org.apache.pulsar.client.impl.ClientBuilderImpl;
 import org.apache.pulsar.client.impl.ClientCnx;
 import org.apache.pulsar.client.impl.ConsumerBase;
 import org.apache.pulsar.client.impl.ConsumerImpl;
@@ -105,6 +107,8 @@
 import org.apache.pulsar.common.compression.CompressionCodec;
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.PublisherStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -4581,4 +4585,32 @@ public void testSendMsgGreaterThanBatchingMaxBytes() throws Exception {
         // sendAsync should complete in time
         assertNotNull(producer.sendAsync(msg).get(timeoutSec, TimeUnit.SECONDS));
     }
-}
\ No newline at end of file
+
+    @Test
+    public void testClientVersion() throws Exception {
+        String defaultClientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion();
+        String topic = "persistent://my-property/my-ns/test-client-version";
+
+        Producer producer1 = pulsarClient.newProducer()
+                .topic(topic)
+                .create();
+        TopicStats stats = admin.topics().getStats(topic);
+        assertEquals(stats.getPublishers().size(), 1);
+        assertEquals(stats.getPublishers().get(0).getClientVersion(), defaultClientVersion);
+
+        PulsarClient client = ((ClientBuilderImpl) PulsarClient.builder())
+                .description("my-java-client")
+                .serviceUrl(lookupUrl.toString())
+                .build();
+        Producer producer2 = client.newProducer().topic(topic).create();
+        stats = admin.topics().getStats(topic);
+        assertEquals(stats.getPublishers().size(), 2);
+
+        assertEquals(stats.getPublishers().stream().map(PublisherStats::getClientVersion).collect(Collectors.toSet()),
+                Sets.newHashSet(defaultClientVersion, defaultClientVersion + "-my-java-client"));
+
+        producer1.close();
+        producer2.close();
+        client.close();
+    }
+}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java
index 523acdace3950..7677045f0899b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java
@@ -410,4 +410,27 @@ public ClientBuilder socks5ProxyPassword(String socks5ProxyPassword) {
         conf.setSocks5ProxyPassword(socks5ProxyPassword);
         return this;
     }
+
+    /**
+     * Set the description.
+     *
+     * 
 By default, when the client connects to the broker, a version string like "Pulsar-Java-v" will be
+     * carried and saved by the broker. The client version string could be queried from the topic stats.
+     *
+     * 
 This method provides a way to add more description to a specific PulsarClient instance. If it's configured,
+     * the description will be appended to the original client version string, with '-' as the separator.
+     *
+     * 
For example, if the client version is 3.0.0, and the description is "forked", the final client version string
+     * will be "Pulsar-Java-v3.0.0-forked".
+     *
+     * @param description the description of the current PulsarClient instance
+     * @throws IllegalArgumentException if the length of description exceeds 64
+     */
+    public ClientBuilder description(String description) {
+        if (description != null && description.length() > 64) {
+            throw new IllegalArgumentException("description should be at most 64 characters");
+        }
+        conf.setDescription(description);
+        return this;
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
index 7780856c6948e..115c71307c4f2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
@@ -194,6 +194,8 @@ public class ClientCnx extends PulsarHandler {
     @Getter
     private long lastDisconnectedTimestamp;
 
+    private final String clientVersion;
+
     protected enum State {
         None, SentConnectFrame, Ready, Failed, Connecting
     }
@@ -252,6 +254,8 @@ public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, in
         this.state = State.None;
         this.protocolVersion = protocolVersion;
         this.idleState = new ClientCnxIdleState(this);
+        this.clientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion()
+                + (conf.getDescription() == null ? "" : ("-" + conf.getDescription()));
     }
 
     @Override
@@ -293,8 +297,7 @@ protected ByteBuf newConnectCommand() throws Exception {
         authenticationDataProvider = authentication.getAuthData(remoteHostName);
         AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA);
         return Commands.newConnect(authentication.getAuthMethodName(), authData, this.protocolVersion,
-                String.format("Pulsar-Java-v%s", PulsarVersion.getVersion()), proxyToTargetBrokerAddress, null, null,
-                null);
+                clientVersion, proxyToTargetBrokerAddress, null, null, null);
     }
 
     @Override
@@ -411,7 +414,7 @@ protected void handleAuthChallenge(CommandAuthChallenge authChallenge) {
             ByteBuf request = Commands.newAuthResponse(authentication.getAuthMethodName(),
                     authData,
                     this.protocolVersion,
-                    String.format("Pulsar-Java-v%s", PulsarVersion.getVersion()));
+                    clientVersion);
 
             if (log.isDebugEnabled()) {
                 log.debug("{} Mutual auth {}", ctx.channel(), authentication.getAuthMethodName());
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java
index 1e7bc6f8221cf..7d94675ccba7d 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java
@@ -379,6 +379,12 @@ public class ClientConfigurationData implements Serializable, Cloneable {
     @Secret
     private String socks5ProxyPassword;
 
+    @ApiModelProperty(
+            name = "description",
+            value = "The extra description of the client version. The length cannot exceed 64."
+    )
+    private String description;
+
     /**
      * Gets the authentication settings for the client.
      *

From 02c838c990543200c05f3d1479d74e685f895bba Mon Sep 17 00:00:00 2001
From: Yan Zhao 
Date: Mon, 10 Apr 2023 22:49:29 +0800
Subject: [PATCH 157/404] [fix][broker]Make LedgerOffloaderFactory can load the
 old nar. (#19913)

After #13833, we change the signature of LedgerOffloaderFactory#create. But some users may use the old version LedgerOffloaderFactory in the offloader nar, when pulsar load the offloader nar, it will use the LedgerOffloaderFactory in the offloader nar, the LedgerOffloaderFactory#create has no param offloaderStats.

Modifications: Make LedgerOffloaderFactory can load the old nar.
---
 .../mledger/LedgerOffloaderFactory.java       | 39 ++++++++++++++++++-
 .../mledger/LedgerOffloaderStatsDisable.java  |  4 +-
 .../FileSystemLedgerOffloaderFactory.java     |  8 ++++
 .../jcloud/JCloudLedgerOffloaderFactory.java  |  7 ++++
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java
index 42f92359f9a94..7ecb8f08d573d 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java
@@ -42,7 +42,7 @@ public interface LedgerOffloaderFactory {
     boolean isDriverSupported(String driverName);
 
     /**
-     * Create a ledger offloader with the provided configuration, user-metadata and scheduler.
+     * Create a ledger offloader with the provided configuration, user-metadata, scheduler and offloaderStats.
      *
      * @param offloadPolicies offload policies
      * @param userMetadata user metadata
@@ -50,12 +50,29 @@ public interface LedgerOffloaderFactory {
      * @return the offloader instance
      * @throws IOException when fail to create an offloader
      */
+    T create(OffloadPoliciesImpl offloadPolicies,
+             Map userMetadata,
+             OrderedScheduler scheduler)
+            throws IOException;
+
+
+    /**
+     * Create a ledger offloader with the provided configuration, user-metadata, scheduler and offloaderStats.
+     *
+     * @param offloadPolicies offload policies
+     * @param userMetadata user metadata
+     * @param scheduler scheduler
+     * @param offloaderStats offloaderStats
+     * @return the offloader instance
+     * @throws IOException when fail to create an offloader
+     */
     T create(OffloadPoliciesImpl offloadPolicies,
              Map userMetadata,
              OrderedScheduler scheduler,
              LedgerOffloaderStats offloaderStats)
         throws IOException;
 
+
     /**
      * Create a ledger offloader with the provided configuration, user-metadata, schema storage and scheduler.
      *
@@ -66,6 +83,26 @@ T create(OffloadPoliciesImpl offloadPolicies,
      * @return the offloader instance
      * @throws IOException when fail to create an offloader
      */
+    default T create(OffloadPoliciesImpl offloadPolicies,
+                     Map userMetadata,
+                     SchemaStorage schemaStorage,
+                     OrderedScheduler scheduler)
+            throws IOException {
+        return create(offloadPolicies, userMetadata, scheduler);
+    }
+
+    /**
+     * Create a ledger offloader with the provided configuration, user-metadata, schema storage,
+     * scheduler and offloaderStats.
+     *
+     * @param offloadPolicies offload policies
+     * @param userMetadata user metadata
+     * @param schemaStorage used for schema lookup in offloader
+     * @param scheduler scheduler
+     * @param offloaderStats offloaderStats
+     * @return the offloader instance
+     * @throws IOException when fail to create an offloader
+     */
     default T create(OffloadPoliciesImpl offloadPolicies,
                      Map userMetadata,
                      SchemaStorage schemaStorage,
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java
index 0fe0f453347bf..eeac9cfcfa994 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderStatsDisable.java
@@ -20,9 +20,9 @@
 
 import java.util.concurrent.TimeUnit;
 
-class LedgerOffloaderStatsDisable implements LedgerOffloaderStats {
+public class LedgerOffloaderStatsDisable implements LedgerOffloaderStats {
 
-    static final LedgerOffloaderStats INSTANCE = new LedgerOffloaderStatsDisable();
+    public static final LedgerOffloaderStats INSTANCE = new LedgerOffloaderStatsDisable();
 
     private LedgerOffloaderStatsDisable() {
 
diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java
index 8853ddf3da8ac..5649d264f579c 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java
@@ -23,6 +23,7 @@
 import org.apache.bookkeeper.common.util.OrderedScheduler;
 import org.apache.bookkeeper.mledger.LedgerOffloaderFactory;
 import org.apache.bookkeeper.mledger.LedgerOffloaderStats;
+import org.apache.bookkeeper.mledger.LedgerOffloaderStatsDisable;
 import org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
 
@@ -32,6 +33,13 @@ public boolean isDriverSupported(String driverName) {
         return FileSystemManagedLedgerOffloader.driverSupported(driverName);
     }
 
+    @Override
+    public FileSystemManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies,
+                                                   Map userMetadata, OrderedScheduler scheduler)
+            throws IOException {
+        return create(offloadPolicies, userMetadata, scheduler, LedgerOffloaderStatsDisable.INSTANCE);
+    }
+
     @Override
     public FileSystemManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies,
                                                    Map userMetadata,
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java
index e10575f5ddbb4..2c9165674444d 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java
@@ -23,6 +23,7 @@
 import org.apache.bookkeeper.common.util.OrderedScheduler;
 import org.apache.bookkeeper.mledger.LedgerOffloaderFactory;
 import org.apache.bookkeeper.mledger.LedgerOffloaderStats;
+import org.apache.bookkeeper.mledger.LedgerOffloaderStatsDisable;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration;
@@ -44,6 +45,12 @@ public boolean isDriverSupported(String driverName) {
         return JCloudBlobStoreProvider.driverSupported(driverName);
     }
 
+    @Override
+    public BlobStoreManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies, Map userMetadata,
+                                                  OrderedScheduler scheduler) throws IOException {
+        return create(offloadPolicies, userMetadata, scheduler, LedgerOffloaderStatsDisable.INSTANCE);
+    }
+
     @Override
     public BlobStoreManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicies, Map userMetadata,
                                                   OrderedScheduler scheduler,

From 7990948a73e2c4dfa0e9c99ff223f0ee90e82dc3 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 10 Apr 2023 12:13:13 -0500
Subject: [PATCH 158/404] [refactor][broker] Use AuthenticationParameters for
 rest producer (#20046)

### Motivation

In #19975, we introduced a wrapper for all authentication parameters. This PR adds that wrapper to the Rest Producer.

### Modifications

* Use `AuthenticationParameters` to simplify parameter management in Rest Producer.
* Add method to the `AuthorizationService` that takes the `AuthenticationParameters`.
* Update annotations on Rest Producer to indicate that a 401 is an expected response.

### Verifying this change

This change is covered by the `TopicsAuthTest`.

### Documentation

- [x] `doc-not-needed`

This is an internal change that does not need to be documented.

### Matching PR in forked repository

PR in forked repository: skipping PR since the relevant tests pass locally
---
 .../authorization/AuthorizationService.java   |  7 ++++++
 .../org/apache/pulsar/broker/rest/Topics.java |  4 +++
 .../apache/pulsar/broker/rest/TopicsBase.java | 25 ++++++++++++++-----
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index 6c730f20092dd..706eadf0ec2dc 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -748,6 +748,13 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName,
         }
     }
 
+    public CompletableFuture allowTopicOperationAsync(TopicName topicName,
+                                                               TopicOperation operation,
+                                                               AuthenticationParameters authParams) {
+        return allowTopicOperationAsync(topicName, operation, authParams.getOriginalPrincipal(),
+                authParams.getClientRole(), authParams.getClientAuthenticationDataSource());
+    }
+
     public CompletableFuture allowTopicOperationAsync(TopicName topicName,
                                                                TopicOperation operation,
                                                                String originalRole,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java
index 5b17b05db209f..f1e7009df0257 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/Topics.java
@@ -49,6 +49,7 @@ public class Topics extends TopicsBase {
     @Path("/persistent/{tenant}/{namespace}/{topic}")
     @ApiOperation(value = "Produce message to a persistent topic.", response = String.class, responseContainer = "List")
     @ApiResponses(value = {
+            @ApiResponse(code = 401, message = "Client is not authorized to perform operation"),
             @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"),
             @ApiResponse(code = 412, message = "Namespace name is not valid"),
             @ApiResponse(code = 500, message = "Internal server error") })
@@ -76,6 +77,7 @@ public void produceOnPersistentTopic(@Suspended final AsyncResponse asyncRespons
     @ApiOperation(value = "Produce message to a partition of a persistent topic.",
             response = String.class, responseContainer = "List")
     @ApiResponses(value = {
+            @ApiResponse(code = 401, message = "Client is not authorized to perform operation"),
             @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"),
             @ApiResponse(code = 412, message = "Namespace name is not valid"),
             @ApiResponse(code = 500, message = "Internal server error") })
@@ -104,6 +106,7 @@ public void produceOnPersistentTopicPartition(@Suspended final AsyncResponse asy
     @Path("/non-persistent/{tenant}/{namespace}/{topic}")
     @ApiOperation(value = "Produce message to a persistent topic.", response = String.class, responseContainer = "List")
     @ApiResponses(value = {
+            @ApiResponse(code = 401, message = "Client is not authorized to perform operation"),
             @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"),
             @ApiResponse(code = 412, message = "Namespace name is not valid"),
             @ApiResponse(code = 500, message = "Internal server error") })
@@ -132,6 +135,7 @@ public void produceOnNonPersistentTopic(@Suspended final AsyncResponse asyncResp
     @ApiOperation(value = "Produce message to a partition of a persistent topic.",
             response = String.class, responseContainer = "List")
     @ApiResponses(value = {
+            @ApiResponse(code = 401, message = "Client is not authorized to perform operation"),
             @ApiResponse(code = 404, message = "tenant/namespace/topic doesn't exit"),
             @ApiResponse(code = 412, message = "Namespace name is not valid"),
             @ApiResponse(code = 500, message = "Internal server error") })
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
index 4614eb59a7016..7d3aa37fa4ec5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.rest;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import com.fasterxml.jackson.databind.ObjectReader;
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
@@ -54,6 +55,7 @@
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase;
+import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.namespace.LookupOptions;
 import org.apache.pulsar.broker.service.BrokerServiceException;
@@ -81,6 +83,7 @@
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
+import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.schema.SchemaData;
 import org.apache.pulsar.common.protocol.schema.SchemaVersion;
@@ -762,14 +765,24 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) {
             if (!isClientAuthenticated(clientAppId())) {
                 throw new RestException(Status.UNAUTHORIZED, "Need to authenticate to perform the request");
             }
+            AuthenticationParameters authParams = authParams();
+            boolean isAuthorized;
+            try {
+                isAuthorized = pulsar().getBrokerService().getAuthorizationService()
+                        .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, authParams)
+                        .get(config().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+            } catch (InterruptedException e) {
+                log.warn("Time-out {} sec while checking authorization on {} ",
+                        config().getMetadataStoreOperationTimeoutSeconds(), topicName);
+                throw new RestException(Status.INTERNAL_SERVER_ERROR, "Time-out while checking authorization");
+            } catch (Exception e) {
+                log.warn("Producer-client  with Role - {} {} failed to get permissions for topic - {}. {}",
+                        authParams.getClientRole(), authParams.getOriginalPrincipal(), topicName, e.getMessage());
+                throw new RestException(Status.INTERNAL_SERVER_ERROR, "Failed to get permissions");
+            }
 
-            boolean isAuthorized = pulsar().getBrokerService().getAuthorizationService()
-                    .canProduce(topicName, originalPrincipal() == null ? clientAppId() : originalPrincipal(),
-                            clientAuthData());
             if (!isAuthorized) {
-                throw new RestException(Status.UNAUTHORIZED, String.format("Unauthorized to produce to topic %s"
-                                        + " with clientAppId [%s] and authdata %s", topicName.toString(),
-                        clientAppId(), clientAuthData()));
+                throw new RestException(Status.UNAUTHORIZED, "Unauthorized to produce to topic " + topicName);
             }
         }
     }

From 25af808970fc02281a0b9345c5de205654d7b789 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Mon, 10 Apr 2023 10:27:58 -0700
Subject: [PATCH 159/404] [improve][client] Add retry when readTailMessages
 fails (#20044)

---
 .../pulsar/client/impl/TableViewTest.java     | 41 +++++++++++++++++++
 .../pulsar/client/impl/TableViewImpl.java     | 29 ++++++-------
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java
index b6569d6a21dc1..6c6da5870aed9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java
@@ -19,6 +19,8 @@
 package org.apache.pulsar.client.impl;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -31,6 +33,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
@@ -40,6 +43,7 @@
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerBuilder;
+import org.apache.pulsar.client.api.Reader;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TableView;
 import org.apache.pulsar.common.naming.TopicDomain;
@@ -397,4 +401,41 @@ public void testTableViewWithEncryptedMessages() throws Exception {
         });
         assertEquals(tv.keySet(), keys);
     }
+
+    @Test(timeOut = 30 * 1000)
+    public void testTableViewTailMessageReadRetry() throws Exception {
+        String topic = "persistent://public/default/tableview-is-interrupted-test";
+        admin.topics().createNonPartitionedTopic(topic);
+        @Cleanup
+        TableView tv = pulsarClient.newTableView(Schema.BYTES)
+                .topic(topic)
+                .autoUpdatePartitionsInterval(60, TimeUnit.SECONDS)
+                .create();
+
+        // inject failure on consumer.receiveAsync()
+        var reader = ((CompletableFuture>)
+                FieldUtils.readDeclaredField(tv, "reader", true)).join();
+        var consumer = spy((ConsumerImpl)
+                FieldUtils.readDeclaredField(reader, "consumer", true));
+
+        var errorCnt = new AtomicInteger(3);
+        doAnswer(invocationOnMock -> {
+            if (errorCnt.decrementAndGet() > 0) {
+                return CompletableFuture.failedFuture(new RuntimeException());
+            }
+            // Call the real method
+            reset(consumer);
+            return consumer.receiveAsync();
+        }).when(consumer).receiveAsync();
+        FieldUtils.writeDeclaredField(reader, "consumer", consumer, true);
+
+        int msgCnt = 2;
+        this.publishMessages(topic, msgCnt, false, false);
+        Awaitility.await()
+                .atMost(5, TimeUnit.SECONDS)
+                .untilAsserted(() -> {
+            assertEquals(tv.size(), msgCnt);
+        });
+        verify(consumer, times(msgCnt)).receiveAsync();
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
index 81771126f76ce..ff5b251ad55ff 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
@@ -240,9 +240,13 @@ private void readAllExistingMessages(Reader reader, CompletableFuture {
-                                   logException(
-                                           String.format("Reader %s was interrupted while reading existing messages",
-                                                   reader.getTopic()), ex);
+                                   if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) {
+                                       log.error("Reader {} was closed while reading existing messages.",
+                                               reader.getTopic(), ex);
+                                   } else {
+                                       log.warn("Reader {} was interrupted while reading existing messages. ",
+                                               reader.getTopic(), ex);
+                                   }
                                    future.completeExceptionally(ex);
                                    return null;
                                });
@@ -266,18 +270,15 @@ private void readTailMessages(Reader reader) {
                     handleMessage(msg);
                     readTailMessages(reader);
                 }).exceptionally(ex -> {
-                    logException(
-                            String.format("Reader %s was interrupted while reading tail messages.",
-                                    reader.getTopic()), ex);
+                    if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) {
+                        log.error("Reader {} was closed while reading tail messages.",
+                                reader.getTopic(), ex);
+                    } else {
+                        log.warn("Reader {} was interrupted while reading tail messages. "
+                                        + "Retrying..", reader.getTopic(), ex);
+                        readTailMessages(reader);
+                    }
                     return null;
                 });
     }
-
-    private void logException(String msg, Throwable ex) {
-        if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) {
-            log.warn(msg, ex);
-        } else {
-            log.error(msg, ex);
-        }
-    }
 }

From a7a605f13565edc380b8ae01808832c426f16d1f Mon Sep 17 00:00:00 2001
From: Hang Chen 
Date: Tue, 11 Apr 2023 02:35:18 +0800
Subject: [PATCH 160/404] [improve] [broker] Upgrade the BookKeeper dependency
 to 4.16.0 (#20049)

---
 .../server/src/assemble/LICENSE.bin.txt       | 54 ++++++++++---------
 .../shell/src/assemble/LICENSE.bin.txt        |  6 +--
 pom.xml                                       |  2 +-
 .../prometheus/metrics/LongAdderCounter.java  |  9 +++-
 .../auth/SameThreadOrderedSafeExecutor.java   |  5 +-
 .../broker/stats/PrometheusMetricsTest.java   | 19 +------
 pulsar-sql/presto-distribution/LICENSE        | 29 +++++-----
 .../presto/PulsarConnectorMetricsTracker.java |  6 +--
 .../bookkeeper/client/TestStatsProvider.java  |  8 ++-
 9 files changed, 70 insertions(+), 68 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 6b3455127b423..9205e072f7efe 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -343,35 +343,37 @@ The Apache Software License, Version 2.0
     - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar
     - org.apache.logging.log4j-log4j-web-2.18.0.jar
  * Java Native Access JNA
-    - net.java.dev.jna-jna-5.12.1.jar
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.15.4.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.4.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.15.4.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.15.4.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.4.jar
-    - org.apache.bookkeeper-circe-checksum-4.15.4.jar
-    - org.apache.bookkeeper-cpu-affinity-4.15.4.jar
-    - org.apache.bookkeeper-statelib-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-api-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-common-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-server-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.15.4.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.15.4.jar
-    - org.apache.bookkeeper.http-http-server-4.15.4.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.15.4.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.4.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.4.jar
-    - org.apache.distributedlog-distributedlog-common-4.15.4.jar
-    - org.apache.distributedlog-distributedlog-core-4.15.4-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.15.4.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.15.4.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar
+    - org.apache.bookkeeper-circe-checksum-4.16.0.jar
+    - org.apache.bookkeeper-cpu-affinity-4.16.0.jar
+    - org.apache.bookkeeper-statelib-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-api-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-common-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-server-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar
+    - org.apache.bookkeeper.http-http-server-4.16.0.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar
+    - org.apache.distributedlog-distributedlog-common-4.16.0.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.0.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar
+    - org.apache.bookkeeper-native-io-4.16.0.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 90896790b1fba..0ac08caa1c01d 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -390,9 +390,9 @@ The Apache Software License, Version 2.0
     - log4j-web-2.18.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.15.4.jar
-    - cpu-affinity-4.15.4.jar
-    - circe-checksum-4.15.4.jar
+    - bookkeeper-common-allocator-4.16.0.jar
+    - cpu-affinity-4.16.0.jar
+    - circe-checksum-4.16.0.jar
   * AirCompressor
      - aircompressor-0.20.jar
  * AsyncHttpClient
diff --git a/pom.xml b/pom.xml
index 7f20d478594ae..d6d7fdf44689d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API.
     
     1.21
 
-    4.15.4
+    4.16.0
     3.8.1
     1.5.0
     1.10.0
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java
index ff9d9302456d1..8ade2bc883f9a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.stats.prometheus.metrics;
 
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.bookkeeper.stats.Counter;
 
@@ -50,10 +51,16 @@ public void dec() {
     }
 
     @Override
-    public void add(long delta) {
+    public void addCount(long delta) {
         counter.add(delta);
     }
 
+    @Override
+    public void addLatency(long eventLatency, TimeUnit unit) {
+        long valueMillis = unit.toMillis(eventLatency);
+        counter.add(valueMillis);
+    }
+
     @Override
     public Long get() {
         return counter.sum();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java
index 5f6a32a61bd25..258188a31f5d0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java
@@ -21,7 +21,6 @@
 import io.netty.util.concurrent.DefaultThreadFactory;
 
 import org.apache.bookkeeper.common.util.OrderedExecutor;
-import org.apache.bookkeeper.common.util.SafeRunnable;
 import org.apache.bookkeeper.stats.NullStatsLogger;
 
 public class SameThreadOrderedSafeExecutor extends OrderedExecutor {
@@ -46,12 +45,12 @@ public void execute(Runnable r) {
     }
 
     @Override
-    public void executeOrdered(int orderingKey, SafeRunnable r) {
+    public void executeOrdered(int orderingKey, Runnable r) {
         r.run();
     }
 
     @Override
-    public void executeOrdered(long orderingKey, SafeRunnable r) {
+    public void executeOrdered(long orderingKey, Runnable r) {
         r.run();
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index 04b0a5509b68c..f5ae8459f1803 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -1322,24 +1322,7 @@ public void testManagedLedgerBookieClientStats() throws Exception {
                 System.out.println(e.getKey() + ": " + e.getValue())
         );
 
-        List cm = (List) metrics.get(keyNameBySubstrings(metrics,
-                "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_completed_tasks"));
-        assertEquals(cm.size(), 1);
-        assertEquals(cm.get(0).tags.get("cluster"), "test");
-
-        cm = (List) metrics.get(
-                keyNameBySubstrings(metrics,
-                        "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_queue"));
-        assertEquals(cm.size(), 1);
-        assertEquals(cm.get(0).tags.get("cluster"), "test");
-
-        cm = (List) metrics.get(
-                keyNameBySubstrings(metrics,
-                        "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_total_tasks"));
-        assertEquals(cm.size(), 1);
-        assertEquals(cm.get(0).tags.get("cluster"), "test");
-
-        cm = (List) metrics.get(
+        List cm = (List) metrics.get(
                 keyNameBySubstrings(metrics,
                         "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_threads"));
         assertEquals(cm.size(), 1);
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 6d60765d459b1..e101191bbf536 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -426,18 +426,21 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
   * Apache Bookkeeper
-    - bookkeeper-common-4.15.4.jar
-    - bookkeeper-common-allocator-4.15.4.jar
-    - bookkeeper-proto-4.15.4.jar
-    - bookkeeper-server-4.15.4.jar
-    - bookkeeper-stats-api-4.15.4.jar
-    - bookkeeper-tools-framework-4.15.4.jar
-    - circe-checksum-4.15.4.jar
-    - codahale-metrics-provider-4.15.4.jar
-    - cpu-affinity-4.15.4.jar
-    - http-server-4.15.4.jar
-    - prometheus-metrics-provider-4.15.4.jar
-    - codahale-metrics-provider-4.15.4.jar
+    - bookkeeper-common-4.16.0.jar
+    - bookkeeper-common-allocator-4.16.0.jar
+    - bookkeeper-proto-4.16.0.jar
+    - bookkeeper-server-4.16.0.jar
+    - bookkeeper-stats-api-4.16.0.jar
+    - bookkeeper-tools-framework-4.16.0.jar
+    - circe-checksum-4.16.0.jar
+    - codahale-metrics-provider-4.16.0.jar
+    - cpu-affinity-4.16.0.jar
+    - http-server-4.16.0.jar
+    - prometheus-metrics-provider-4.16.0.jar
+    - codahale-metrics-provider-4.16.0.jar
+    - bookkeeper-slogger-api-4.16.0.jar
+    - bookkeeper-slogger-slf4j-4.16.0.jar
+    - native-io-4.16.0.jar
   * Apache Commons
     - commons-cli-1.5.0.jar
     - commons-codec-1.15.jar
@@ -479,6 +482,8 @@ The Apache Software License, Version 2.0
     - amqp-client-5.5.3.jar
   * Stream Lib
     - stream-2.9.5.jar
+  * High Performance Primitive Collections for Java
+    - hppc-0.9.1.jar
 
 
 Protocol Buffers License
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java
index b586063cc3c9a..12ee2da463c40 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java
@@ -185,7 +185,7 @@ public void end_ENTRY_QUEUE_DEQUEUE_WAIT_TIME() {
     public void register_BYTES_READ(long bytes) {
         if (statsLogger != null) {
             bytesReadSum += bytes;
-            statsLoggerBytesRead.add(bytes);
+            statsLoggerBytesRead.addCount(bytes);
         }
     }
 
@@ -220,7 +220,7 @@ public void end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME() {
     public void incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY() {
         if (statsLogger != null) {
             numMessagedDerserializedPerBatch++;
-            statsLoggerNumMessagesDeserialized.add(1);
+            statsLoggerNumMessagesDeserialized.addCount(1);
         }
     }
 
@@ -295,7 +295,7 @@ public void end_RECORD_DESERIALIZE_TIME() {
 
     public void incr_NUM_RECORD_DESERIALIZED() {
         if (statsLogger != null) {
-            statsLoggerNumRecordDeserialized.add(1);
+            statsLoggerNumRecordDeserialized.addCount(1);
         }
     }
 
diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java
index f015306cb048c..4d08a7f80df5b 100644
--- a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java
+++ b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java
@@ -59,10 +59,16 @@ public void dec() {
         }
 
         @Override
-        public void add(long delta) {
+        public void addCount(long delta) {
             updateMax(val.addAndGet(delta));
         }
 
+        @Override
+        public void addLatency(long eventLatency, TimeUnit unit) {
+            long valueMillis = unit.toMillis(eventLatency);
+            updateMax(val.addAndGet(valueMillis));
+        }
+
         @Override
         public Long get() {
             return val.get();

From c3a92b213f04269678b68e5fe4b72a655955be1a Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 11 Apr 2023 02:44:03 +0800
Subject: [PATCH 161/404] [fix] [ml] Fix uncompleted future when remove cursor
 (#20050)

---
 .../mledger/impl/MetaStoreImpl.java           | 20 ++++++++++---------
 .../mledger/impl/ManagedCursorTest.java       | 17 ++++++++++++++++
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index 3bdaac1e8d80e..fb93e929f50c7 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -302,16 +302,18 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal
                     }
                     callback.operationComplete(null, null);
                 }, executor.chooseThread(ledgerName))
-                .exceptionallyAsync(ex -> {
-                    Throwable actEx = FutureUtil.unwrapCompletionException(ex);
-                    if (actEx instanceof MetadataStoreException.NotFoundException){
-                        log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName);
-                        callback.operationComplete(null, null);
-                        return null;
-                    }
-                    SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex)));
+                .exceptionally(ex -> {
+                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> {
+                        Throwable actEx = FutureUtil.unwrapCompletionException(ex);
+                        if (actEx instanceof MetadataStoreException.NotFoundException){
+                            log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName);
+                            callback.operationComplete(null, null);
+                            return;
+                        }
+                        callback.operationFailed(getException(ex));
+                    }));
                     return null;
-                }, executor.chooseThread(ledgerName));
+                });
     }
 
     @Override
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 8dc726c249efc..1b1b5534256f9 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -48,6 +48,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -95,6 +96,7 @@
 import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
 import org.apache.pulsar.common.api.proto.CommandSubscribe;
 import org.apache.pulsar.common.api.proto.IntRange;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.BitSetRecyclable;
 import org.apache.pulsar.common.util.collections.LongPairRangeSet;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
@@ -1060,6 +1062,21 @@ void removingCursor() throws Exception {
         Awaitility.await().until(() -> ledger.getNumberOfEntries() <= 2);
     }
 
+    @Test(timeOut = 10000)
+    void testRemoveCursorFail() throws Exception {
+        String mlName = UUID.randomUUID().toString().replaceAll("-", "");
+        String cursorName = "c1";
+        ManagedLedger ledger = factory.open(mlName);
+        ledger.openCursor(cursorName);
+        metadataStore.setAlwaysFail(new MetadataStoreException("123"));
+        try {
+            ledger.deleteCursor(cursorName);
+            fail("expected delete cursor failure.");
+        } catch (Exception ex) {
+            assertTrue(FutureUtil.unwrapCompletionException(ex).getMessage().contains("123"));
+        }
+    }
+
     @Test(timeOut = 20000)
     void cursorPersistence() throws Exception {
         ManagedLedger ledger = factory.open("my_test_ledger");

From 34b6e89269619afefa8120bdea3821d6b3811757 Mon Sep 17 00:00:00 2001
From: vineeth1995 
Date: Mon, 10 Apr 2023 11:56:03 -0700
Subject: [PATCH 162/404] [feat] [broker] PIP-188 support blue-green cluster
 migration [part-2] (#19605)

Co-authored-by: Vineeth 
---
 .../pulsar/broker/service/Replicator.java     |   2 +
 .../pulsar/broker/service/ServerCnx.java      |  24 +-
 .../apache/pulsar/broker/service/Topic.java   |   2 +
 .../NonPersistentReplicator.java              |   2 +-
 .../nonpersistent/NonPersistentTopic.java     |   5 +
 .../persistent/PersistentReplicator.java      |   4 +-
 .../service/persistent/PersistentTopic.java   |  21 +-
 .../auth/MockedPulsarServiceBaseTest.java     |   8 +-
 .../broker/service/ClusterMigrationTest.java  | 229 +++++++++++++++---
 9 files changed, 253 insertions(+), 44 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
index b8d13256d225a..482fa2cbd2300 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
@@ -49,4 +49,6 @@ default Optional getRateLimiter() {
     }
 
     boolean isConnected();
+
+    long getNumberOfEntriesInBacklog();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 973dec8980ea0..1b376492accc5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -1579,13 +1579,23 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ
             if (ex.getCause() instanceof BrokerServiceException.TopicMigratedException) {
                 Optional clusterURL = getMigratedClusterUrl(service.getPulsar());
                 if (clusterURL.isPresent()) {
-                    log.info("[{}] redirect migrated producer to topic {}: producerId={}, {}", remoteAddress, topicName,
-                            producerId, ex.getCause().getMessage());
-                    commandSender.sendTopicMigrated(ResourceType.Producer, producerId,
-                            clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls());
-                    closeProducer(producer);
-                    return null;
-
+                    if (topic.isReplicationBacklogExist()) {
+                        log.info("Topic {} is migrated but replication backlog exist: "
+                                        + "producerId = {}, producerName = {}, {}", topicName,
+                                producerId, producerName, ex.getCause().getMessage());
+                    } else {
+                        log.info("[{}] redirect migrated producer to topic {}: "
+                                        + "producerId={}, producerName = {}, {}", remoteAddress,
+                                topicName, producerId, producerName, ex.getCause().getMessage());
+                        boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId,
+                                clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls());
+                        if (!msgSent) {
+                            log.info("client doesn't support topic migration handling {}-{}-{}", topic,
+                                    remoteAddress, producerId);
+                        }
+                        closeProducer(producer);
+                        return null;
+                    }
                 } else {
                     log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}",
                             remoteAddress, topicName, producerId, ex.getCause().getMessage());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
index e6a29368dbb85..7657d77e1299f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
@@ -234,6 +234,8 @@ CompletableFuture createSubscription(String subscriptionName, Init
 
     boolean isBrokerPublishRateExceeded();
 
+    boolean isReplicationBacklogExist();
+
     void disableCnxAutoRead();
 
     void enableCnxAutoRead();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
index 7ad231926189b..514db4219db98 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
@@ -248,7 +248,7 @@ protected Position getReplicatorReadPosition() {
     }
 
     @Override
-    protected long getNumberOfEntriesInBacklog() {
+    public long getNumberOfEntriesInBacklog() {
         // No-op
         return 0;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index a0a8462a22753..317b8df6b9a82 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -236,6 +236,11 @@ public void checkMessageDeduplicationInfo() {
         // No-op
     }
 
+    @Override
+    public boolean isReplicationBacklogExist() {
+        return false;
+    }
+
     @Override
     public void removeProducer(Producer producer) {
         checkArgument(producer.getTopic() == this);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
index f38bcc71582a1..a556237f4342c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
@@ -165,8 +165,8 @@ protected Position getReplicatorReadPosition() {
     }
 
     @Override
-    protected long getNumberOfEntriesInBacklog() {
-        return cursor.getNumberOfEntriesInBacklog(false);
+    public long getNumberOfEntriesInBacklog() {
+        return cursor.getNumberOfEntriesInBacklog(true);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 18a662c4b7a38..0374fc98212f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -575,6 +575,7 @@ public void addComplete(Position pos, ByteBuf entryData, Object ctx) {
 
     @Override
     public synchronized void addFailed(ManagedLedgerException exception, Object ctx) {
+        PublishContext callback = (PublishContext) ctx;
         if (exception instanceof ManagedLedgerFencedException) {
             // If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen
             close();
@@ -587,7 +588,11 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx)
                 List> futures = new ArrayList<>();
                 // send migration url metadata to producers before disconnecting them
                 if (isMigrated()) {
-                    producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl()));
+                    if (isReplicationBacklogExist()) {
+                        log.info("Topic {} is migrated but replication backlog exists. Closing producers.", topic);
+                    } else {
+                        producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl()));
+                    }
                 }
                 producers.forEach((__, producer) -> futures.add(producer.disconnect()));
                 disconnectProducersFuture = FutureUtil.waitForAll(futures);
@@ -599,8 +604,6 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx)
                 return null;
             });
 
-            PublishContext callback = (PublishContext) ctx;
-
             if (exception instanceof ManagedLedgerAlreadyClosedException) {
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] Failed to persist msg in store: {}", topic, exception.getMessage());
@@ -2510,6 +2513,18 @@ public CompletableFuture checkClusterMigration() {
         }
     }
 
+    public boolean isReplicationBacklogExist() {
+        ConcurrentOpenHashMap replicators = getReplicators();
+        if (replicators != null) {
+            for (Replicator replicator : replicators.values()) {
+                if (replicator.getNumberOfEntriesInBacklog() != 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     @Override
     public void checkGC() {
         if (!isDeleteWhileInactive()) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
index cd31f9150e619..3fe8c22b1c4de 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
@@ -112,7 +112,7 @@ public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport {
     protected URI lookupUrl;
 
     protected boolean isTcpLookup = false;
-    protected static final String configClusterName = "test";
+    protected String configClusterName = "test";
 
     protected boolean enableBrokerInterceptor = false;
 
@@ -120,6 +120,12 @@ public MockedPulsarServiceBaseTest() {
         resetConfig();
     }
 
+    protected void setupWithClusterName(String clusterName) throws Exception {
+        this.conf.setClusterName(clusterName);
+        this.configClusterName = clusterName;
+        this.internalSetup();
+    }
+
     protected PulsarService getPulsar() {
         return pulsar;
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
index f376ea4541737..df4f66c43d2b4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
@@ -18,10 +18,12 @@
  */
 package org.apache.pulsar.broker.service;
 
+import static java.lang.Thread.sleep;
 import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
 import java.lang.reflect.Method;
@@ -59,7 +61,8 @@ public class ClusterMigrationTest {
     protected String methodName;
 
     String namespace = "pulsar/migrationNs";
-    TestBroker broker1, broker2;
+    TestBroker broker1, broker2, broker3, broker4;
+
     URL url1;
     URL urlTls1;
     PulsarService pulsar1;
@@ -71,6 +74,16 @@ public class ClusterMigrationTest {
     PulsarService pulsar2;
     PulsarAdmin admin2;
 
+    URL url3;
+    URL urlTls3;
+    PulsarService pulsar3;
+    PulsarAdmin admin3;
+
+    URL url4;
+    URL urlTls4;
+    PulsarService pulsar4;
+    PulsarAdmin admin4;
+
     @DataProvider(name = "TopicsubscriptionTypes")
     public Object[][] subscriptionTypes() {
         return new Object[][] {
@@ -91,9 +104,10 @@ public void setup() throws Exception {
 
         log.info("--- Starting ReplicatorTestBase::setup ---");
 
-        broker1 = new TestBroker();
-        broker2 = new TestBroker();
-        String clusterName = broker1.getClusterName();
+        broker1 = new TestBroker("r1");
+        broker2 = new TestBroker("r2");
+        broker3 = new TestBroker("r3");
+        broker4 = new TestBroker("r4");
 
         pulsar1 = broker1.getPulsarService();
         url1 = new URL(pulsar1.getWebServiceAddress());
@@ -105,32 +119,81 @@ public void setup() throws Exception {
         urlTls2 = new URL(pulsar2.getWebServiceAddressTls());
         admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();
 
-        // Start region 3
+        pulsar3 = broker3.getPulsarService();
+        url3 = new URL(pulsar3.getWebServiceAddress());
+        urlTls3 = new URL(pulsar3.getWebServiceAddressTls());
+        admin3 = PulsarAdmin.builder().serviceHttpUrl(url3.toString()).build();
 
-        // Provision the global namespace
-        admin1.clusters().createCluster(clusterName,
+        pulsar4 = broker4.getPulsarService();
+        url4 = new URL(pulsar4.getWebServiceAddress());
+        urlTls4 = new URL(pulsar4.getWebServiceAddressTls());
+        admin4 = PulsarAdmin.builder().serviceHttpUrl(url4.toString()).build();
+
+
+        admin1.clusters().createCluster("r1",
                 ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString())
                         .brokerServiceUrl(pulsar1.getBrokerServiceUrl())
                         .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build());
-        admin2.clusters().createCluster(clusterName,
+        admin3.clusters().createCluster("r1",
+                ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString())
+                        .brokerServiceUrl(pulsar1.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build());
+        admin2.clusters().createCluster("r2",
+                ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString())
+                        .brokerServiceUrl(pulsar2.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build());
+        admin4.clusters().createCluster("r2",
                 ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString())
                         .brokerServiceUrl(pulsar2.getBrokerServiceUrl())
                         .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build());
 
+        admin1.clusters().createCluster("r3",
+                ClusterData.builder().serviceUrl(url3.toString()).serviceUrlTls(urlTls3.toString())
+                        .brokerServiceUrl(pulsar3.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()).build());
+
+        admin3.clusters().createCluster("r3",
+                ClusterData.builder().serviceUrl(url3.toString()).serviceUrlTls(urlTls3.toString())
+                        .brokerServiceUrl(pulsar3.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()).build());
+
+        admin2.clusters().createCluster("r4",
+                ClusterData.builder().serviceUrl(url4.toString()).serviceUrlTls(urlTls4.toString())
+                        .brokerServiceUrl(pulsar4.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build());
+        admin4.clusters().createCluster("r4",
+                ClusterData.builder().serviceUrl(url4.toString()).serviceUrlTls(urlTls4.toString())
+                        .brokerServiceUrl(pulsar4.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build());
+
+        // Setting r3 as replication cluster for r1
         admin1.tenants().createTenant("pulsar",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet(clusterName)));
-        admin1.namespaces().createNamespace(namespace, Sets.newHashSet(clusterName));
-
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3")));
+        admin3.tenants().createTenant("pulsar",
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3")));
+        admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1", "r3"));
+        admin3.namespaces().createNamespace(namespace);
+        admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r3"));
+
+        // Setting r4 as replication cluster for r2
         admin2.tenants().createTenant("pulsar",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet(clusterName)));
-        admin2.namespaces().createNamespace(namespace, Sets.newHashSet(clusterName));
-
-        assertEquals(admin1.clusters().getCluster(clusterName).getServiceUrl(), url1.toString());
-        assertEquals(admin2.clusters().getCluster(clusterName).getServiceUrl(), url2.toString());
-        assertEquals(admin1.clusters().getCluster(clusterName).getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
-        assertEquals(admin2.clusters().getCluster(clusterName).getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
-
-        Thread.sleep(100);
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4")));
+        admin4.tenants().createTenant("pulsar",
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4")));
+        admin2.namespaces().createNamespace(namespace, Sets.newHashSet("r2", "r4"));
+        admin4.namespaces().createNamespace(namespace);
+        admin2.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r2", "r4"));
+
+        assertEquals(admin1.clusters().getCluster("r1").getServiceUrl(), url1.toString());
+        assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
+        assertEquals(admin3.clusters().getCluster("r3").getServiceUrl(), url3.toString());
+        assertEquals(admin4.clusters().getCluster("r4").getServiceUrl(), url4.toString());
+        assertEquals(admin1.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
+        assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
+        assertEquals(admin3.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
+        assertEquals(admin4.clusters().getCluster("r4").getBrokerServiceUrl(), pulsar4.getBrokerServiceUrl());
+
+        sleep(100);
         log.info("--- ReplicatorTestBase::setup completed ---");
 
     }
@@ -140,6 +203,8 @@ protected void cleanup() throws Exception {
         log.info("--- Shutting down ---");
         broker1.cleanup();
         broker2.cleanup();
+        broker3.cleanup();
+        broker4.cleanup();
     }
 
     @BeforeMethod(alwaysRun = true)
@@ -154,20 +219,19 @@ public void beforeMethod(Method m) throws Exception {
      * (3) Migrate topic to cluster-2
      * (4) Validate producer-1 is connected to cluster-2
      * (5) create consumer1, drain backlog and migrate and reconnect to cluster-2
-     * (6) Create new consumer2 with different subscription on cluster-1, 
+     * (6) Create new consumer2 with different subscription on cluster-1,
      *     which immediately migrate and reconnect to cluster-2
      * (7) Create producer-2 directly to cluster-2
-     * (8) Create producer-3 on cluster-1 which should be redirected to cluster-2 
+     * (8) Create producer-3 on cluster-1 which should be redirected to cluster-2
      * (8) Publish messages using producer1, producer2, and producer3
      * (9) Consume all messages by both consumer1 and consumer2
-     * (10) Create Producer/consumer on non-migrated cluster and verify their connection with cluster-1 
-     * (11) Restart Broker-1 and connect producer/consumer on cluster-1 
+     * (10) Create Producer/consumer on non-migrated cluster and verify their connection with cluster-1
+     * (11) Restart Broker-1 and connect producer/consumer on cluster-1
      * @throws Exception
      */
     @Test(dataProvider = "TopicsubscriptionTypes")
     public void testClusterMigration(boolean persistent, SubscriptionType subType) throws Exception {
         log.info("--- Starting ReplicatorTest::testClusterMigration ---");
-        persistent = false;
         final String topicName = BrokerTestUtil
                 .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic");
 
@@ -202,7 +266,7 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t
         assertFalse(topic2.getProducers().isEmpty());
 
         ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls());
-        admin1.clusters().updateClusterMigration(broker2.getClusterName(), true, migratedUrl);
+        admin1.clusters().updateClusterMigration("r1", true, migratedUrl);
 
         retryStrategically((test) -> {
             try {
@@ -214,12 +278,16 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t
             return false;
         }, 10, 500);
 
+
         topic1.checkClusterMigration().get();
 
+        log.info("before sending message");
+        sleep(1000);
         producer1.sendAsync("test1".getBytes());
 
         // producer is disconnected from cluster-1
         retryStrategically((test) -> topic1.getProducers().isEmpty(), 10, 500);
+        log.info("before asserting");
         assertTrue(topic1.getProducers().isEmpty());
 
         // create 3rd producer on cluster-1 which should be redirected to cluster-2
@@ -297,15 +365,116 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t
         log.info("Successfully consumed messages by migrated consumers");
     }
 
+    @Test(dataProvider = "TopicsubscriptionTypes")
+    public void testClusterMigrationWithReplicationBacklog(boolean persistent, SubscriptionType subType) throws Exception {
+        log.info("--- Starting ReplicatorTest::testClusterMigrationWithReplicationBacklog ---");
+        persistent = true;
+        final String topicName = BrokerTestUtil
+                .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic");
+
+        @Cleanup
+        PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        @Cleanup
+        PulsarClient client3 = PulsarClient.builder().serviceUrl(url3.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        // cluster-1 producer/consumer
+        Producer producer1 = client1.newProducer().topic(topicName).enableBatching(false)
+                .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
+        Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(subType)
+                .subscriptionName("s1").subscribe();
+
+        // cluster-3 consumer
+        Consumer consumer3 = client3.newConsumer().topic(topicName).subscriptionType(subType)
+                .subscriptionName("s1").subscribe();
+        AbstractTopic topic1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get();
+        retryStrategically((test) -> !topic1.getProducers().isEmpty(), 5, 500);
+        retryStrategically((test) -> !topic1.getSubscriptions().isEmpty(), 5, 500);
+        assertFalse(topic1.getProducers().isEmpty());
+        assertFalse(topic1.getSubscriptions().isEmpty());
+
+        // build backlog
+        consumer1.close();
+        retryStrategically((test) -> topic1.getReplicators().size() == 1, 10, 3000);
+        assertEquals(topic1.getReplicators().size(), 1);
+
+       // stop service in the replication cluster to build replication backlog
+        broker3.cleanup();
+        retryStrategically((test) -> broker3.getPulsarService() == null, 10, 1000);
+        assertNull(pulsar3.getBrokerService());
+
+        //publish messages into topic in "r1" cluster
+        int n = 5;
+        for (int i = 0; i < n; i++) {
+            producer1.send("test1".getBytes());
+        }
+        retryStrategically((test) -> topic1.isReplicationBacklogExist(), 10, 1000);
+        assertTrue(topic1.isReplicationBacklogExist());
+
+        @Cleanup
+        PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        // cluster-2 producer/consumer
+        Producer producer2 = client2.newProducer().topic(topicName).enableBatching(false)
+                .producerName("cluster2-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
+        AbstractTopic topic2 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName, false).getNow(null).get();
+        log.info("name of topic 2 - {}", topic2.getName());
+        assertFalse(topic2.getProducers().isEmpty());
+
+        retryStrategically((test) -> topic2.getReplicators().size() == 1, 10, 2000);
+        log.info("replicators should be ready");
+        ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls());
+        admin1.clusters().updateClusterMigration("r1", true, migratedUrl);
+        log.info("update cluster migration called");
+        retryStrategically((test) -> {
+            try {
+                topic1.checkClusterMigration().get();
+                return true;
+            } catch (Exception e) {
+                // ok
+            }
+            return false;
+        }, 10, 500);
+
+        topic1.checkClusterMigration().get();
+
+        producer1.sendAsync("test1".getBytes());
+
+        // producer is disconnected from cluster-1
+        retryStrategically((test) -> topic1.getProducers().isEmpty(), 10, 500);
+        assertTrue(topic1.getProducers().isEmpty());
+
+        // verify that the disconnected producer is not redirected
+        // to replication cluster since there is replication backlog.
+        assertEquals(topic2.getProducers().size(), 1);
+
+        // Restart the service in cluster "r3".
+        broker3.restart();
+        retryStrategically((test) -> broker3.getPulsarService() != null, 10, 1000);
+        assertNotNull(broker3.getPulsarService());
+        pulsar3 = broker3.getPulsarService();
+
+        // verify that the replication backlog drains once service in cluster "r3" is restarted.
+        retryStrategically((test) -> !topic1.isReplicationBacklogExist(), 10, 1000);
+        assertFalse(topic1.isReplicationBacklogExist());
+
+        // verify that the producer1 is now is now connected to migrated cluster "r2" since backlog is cleared.
+        retryStrategically((test) -> topic2.getProducers().size()==2, 10, 500);
+        assertEquals(topic2.getProducers().size(), 2);
+    }
+
     static class TestBroker extends MockedPulsarServiceBaseTest {
 
-        public TestBroker() throws Exception {
+        private String clusterName;
+
+        public TestBroker(String clusterName) throws Exception {
+            this.clusterName = clusterName;
             setup();
         }
 
         @Override
         protected void setup() throws Exception {
-            super.internalSetup();
+            super.setupWithClusterName(clusterName);
         }
 
         public PulsarService getPulsarService() {
@@ -318,9 +487,9 @@ public String getClusterName() {
 
         @Override
         protected void cleanup() throws Exception {
-            internalCleanup();
+            stopBroker();
         }
-        
+
         public void restart() throws Exception {
             restartBroker();
         }

From 08b28f5c5cbbdc63467455c40106173475ff01e5 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 10 Apr 2023 22:56:11 +0300
Subject: [PATCH 163/404] [improve][broker] Prevent range conflicts with Key
 Shared sticky consumers when TCP/IP connections get orphaned (#20026)

---
 .../pulsar/broker/ServiceConfiguration.java   |   9 +
 ...bstractDispatcherSingleActiveConsumer.java |  10 +-
 ...stentHashingStickyKeyConsumerSelector.java |   5 +-
 .../pulsar/broker/service/Dispatcher.java     |   2 +-
 ...ngeAutoSplitStickyKeyConsumerSelector.java |  11 +-
 ...ngeExclusiveStickyKeyConsumerSelector.java |  53 +++--
 .../pulsar/broker/service/ServerCnx.java      |  48 ++++-
 .../service/StickyKeyConsumerSelector.java    |   4 +-
 .../pulsar/broker/service/TransportCnx.java   |   9 +
 ...PersistentDispatcherMultipleConsumers.java |   8 +-
 ...tStickyKeyDispatcherMultipleConsumers.java |  26 ++-
 .../NonPersistentSubscription.java            |   7 +-
 ...PersistentDispatcherMultipleConsumers.java |   9 +-
 ...tStickyKeyDispatcherMultipleConsumers.java |  50 +++--
 .../persistent/PersistentSubscription.java    |   7 +-
 .../service/AbstractBaseDispatcherTest.java   |   4 +-
 ...xclusiveStickyKeyConsumerSelectorTest.java |  81 +++++---
 .../pulsar/common/protocol/PulsarHandler.java |  23 ++-
 pulsar-proxy/pom.xml                          |   5 +
 .../proxy/server/LookupProxyHandler.java      |   7 +-
 .../pulsar/proxy/server/ProxyConnection.java  |   2 +-
 .../pulsar/proxy/server/ProxyService.java     |   4 +
 .../server/ProxyStuckConnectionTest.java      | 194 ++++++++++++++++++
 23 files changed, 463 insertions(+), 115 deletions(-)
 create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index c86358b9b5b9e..cea159b74fd1d 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2676,6 +2676,15 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
         doc = "How often to check pulsar connection is still alive"
     )
     private int keepAliveIntervalSeconds = 30;
+    @FieldContext(
+            category = CATEGORY_SERVER,
+            doc = "Timeout for connection liveness check used to check liveness of possible consumer or producer "
+                    + "duplicates. Helps prevent ProducerFencedException with exclusive producer, "
+                    + "ConsumerAssignException with range conflict for Key Shared with sticky hash ranges or "
+                    + "ConsumerBusyException in the case of an exclusive consumer. Set to 0 to disable connection "
+                    + "liveness check."
+    )
+    private long connectionLivenessCheckTimeoutMillis = 5000L;
     @Deprecated
     @FieldContext(
         category = CATEGORY_POLICIES,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java
index 1b6df5f809327..5098890242b6c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java
@@ -36,6 +36,7 @@
 import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException;
 import org.apache.pulsar.client.impl.Murmur3Hash32;
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.Murmur3_32Hash;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -157,21 +158,21 @@ private NavigableMap makeHashRing(int consumerSize) {
         return Collections.unmodifiableNavigableMap(hashRing);
     }
 
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
         if (IS_CLOSED_UPDATER.get(this) == TRUE) {
             log.warn("[{}] Dispatcher is already closed. Closing consumer {}", this.topicName, consumer);
             consumer.disconnect();
-            return;
+            return CompletableFuture.completedFuture(null);
         }
 
         if (subscriptionType == SubType.Exclusive && !consumers.isEmpty()) {
-            throw new ConsumerBusyException("Exclusive consumer is already connected");
+            return FutureUtil.failedFuture(new ConsumerBusyException("Exclusive consumer is already connected"));
         }
 
         if (subscriptionType == SubType.Failover && isConsumersExceededOnSubscription()) {
             log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit",
                     this.topicName);
-            throw new ConsumerBusyException("Subscription reached max consumers limit");
+            return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit"));
         }
 
         if (subscriptionType == SubType.Exclusive
@@ -203,6 +204,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce
             }
         }
 
+        return CompletableFuture.completedFuture(null);
     }
 
     public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceException {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java
index 30dc1248d0c7d..ea491bd40d332 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java
@@ -26,9 +26,9 @@
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.TreeMap;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException;
 import org.apache.pulsar.client.api.Range;
 import org.apache.pulsar.common.util.Murmur3_32Hash;
 
@@ -53,7 +53,7 @@ public ConsistentHashingStickyKeyConsumerSelector(int numberOfPoints) {
     }
 
     @Override
-    public void addConsumer(Consumer consumer) throws ConsumerAssignException {
+    public CompletableFuture addConsumer(Consumer consumer) {
         rwLock.writeLock().lock();
         try {
             // Insert multiple points on the hash ring for every consumer
@@ -73,6 +73,7 @@ public void addConsumer(Consumer consumer) throws ConsumerAssignException {
                     }
                 });
             }
+            return CompletableFuture.completedFuture(null);
         } finally {
             rwLock.writeLock().unlock();
         }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
index 9b0c4e885e64d..3ca06dc83d9aa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
@@ -27,7 +27,7 @@
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 
 public interface Dispatcher {
-    void addConsumer(Consumer consumer) throws BrokerServiceException;
+    CompletableFuture addConsumer(Consumer consumer);
 
     void removeConsumer(Consumer consumer) throws BrokerServiceException;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java
index 59b815197694e..a9fea5b39bf82 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeAutoSplitStickyKeyConsumerSelector.java
@@ -23,9 +23,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException;
 import org.apache.pulsar.client.api.Range;
+import org.apache.pulsar.common.util.FutureUtil;
 
 /**
  * This is a consumer selector based fixed hash range.
@@ -77,13 +79,18 @@ public HashRangeAutoSplitStickyKeyConsumerSelector(int rangeSize) {
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws ConsumerAssignException {
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
         if (rangeMap.isEmpty()) {
             rangeMap.put(rangeSize, consumer);
             consumerRange.put(consumer, rangeSize);
         } else {
-            splitRange(findBiggestRange(), consumer);
+            try {
+                splitRange(findBiggestRange(), consumer);
+            } catch (ConsumerAssignException e) {
+                return FutureUtil.failedFuture(e);
+            }
         }
+        return CompletableFuture.completedFuture(null);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java
index 45be6d65b09b4..78bad1b2c400e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelector.java
@@ -23,10 +23,12 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.pulsar.client.api.Range;
 import org.apache.pulsar.common.api.proto.IntRange;
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
+import org.apache.pulsar.common.util.FutureUtil;
 
 /**
  * This is a sticky-key consumer selector based user provided range.
@@ -52,8 +54,23 @@ public HashRangeExclusiveStickyKeyConsumerSelector(int rangeSize) {
     }
 
     @Override
-    public void addConsumer(Consumer consumer) throws BrokerServiceException.ConsumerAssignException {
-        validateKeySharedMeta(consumer);
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
+        return validateKeySharedMeta(consumer).thenRun(() -> {
+            try {
+                internalAddConsumer(consumer);
+            } catch (BrokerServiceException.ConsumerAssignException e) {
+                throw FutureUtil.wrapToCompletionException(e);
+            }
+        });
+    }
+
+    private synchronized void internalAddConsumer(Consumer consumer)
+            throws BrokerServiceException.ConsumerAssignException {
+        Consumer conflictingConsumer = findConflictingConsumer(consumer.getKeySharedMeta().getHashRangesList());
+        if (conflictingConsumer != null) {
+            throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer "
+                    + conflictingConsumer);
+        }
         for (IntRange intRange : consumer.getKeySharedMeta().getHashRangesList()) {
             rangeMap.put(intRange.getStart(), consumer);
             rangeMap.put(intRange.getEnd(), consumer);
@@ -101,31 +118,41 @@ public Consumer select(int hash) {
         }
     }
 
-    private void validateKeySharedMeta(Consumer consumer) throws BrokerServiceException.ConsumerAssignException {
+    private synchronized CompletableFuture validateKeySharedMeta(Consumer consumer) {
         if (consumer.getKeySharedMeta() == null) {
-            throw new BrokerServiceException.ConsumerAssignException("Must specify key shared meta for consumer.");
+            return FutureUtil.failedFuture(
+                    new BrokerServiceException.ConsumerAssignException("Must specify key shared meta for consumer."));
         }
         List ranges = consumer.getKeySharedMeta().getHashRangesList();
         if (ranges.isEmpty()) {
-            throw new BrokerServiceException.ConsumerAssignException("Ranges for KeyShared policy must not be empty.");
+            return FutureUtil.failedFuture(new BrokerServiceException.ConsumerAssignException(
+                    "Ranges for KeyShared policy must not be empty."));
         }
         for (IntRange intRange : ranges) {
-
             if (intRange.getStart() > intRange.getEnd()) {
-                throw new BrokerServiceException.ConsumerAssignException("Fixed hash range start > end");
+                return FutureUtil.failedFuture(
+                        new BrokerServiceException.ConsumerAssignException("Fixed hash range start > end"));
             }
+        }
+        Consumer conflictingConsumer = findConflictingConsumer(ranges);
+        if (conflictingConsumer != null) {
+            return conflictingConsumer.cnx().checkConnectionLiveness().thenRun(() -> {});
+        } else {
+            return CompletableFuture.completedFuture(null);
+        }
+    }
 
+    private synchronized Consumer findConflictingConsumer(List ranges) {
+        for (IntRange intRange : ranges) {
             Map.Entry ceilingEntry = rangeMap.ceilingEntry(intRange.getStart());
             Map.Entry floorEntry = rangeMap.floorEntry(intRange.getEnd());
 
             if (floorEntry != null && floorEntry.getKey() >= intRange.getStart()) {
-                throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer "
-                        + floorEntry.getValue());
+                return floorEntry.getValue();
             }
 
             if (ceilingEntry != null && ceilingEntry.getKey() <= intRange.getEnd()) {
-                throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer "
-                        + ceilingEntry.getValue());
+                return ceilingEntry.getValue();
             }
 
             if (ceilingEntry != null && floorEntry != null && ceilingEntry.getValue().equals(floorEntry.getValue())) {
@@ -134,12 +161,12 @@ private void validateKeySharedMeta(Consumer consumer) throws BrokerServiceExcept
                     int start = Math.max(intRange.getStart(), range.getStart());
                     int end = Math.min(intRange.getEnd(), range.getEnd());
                     if (end >= start) {
-                        throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer "
-                                + ceilingEntry.getValue());
+                        return ceilingEntry.getValue();
                     }
                 }
             }
         }
+        return null;
     }
 
     Map getRangeConsumer() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 1b376492accc5..677ed25ef2163 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -164,6 +164,7 @@
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
 import org.apache.pulsar.common.util.netty.NettyChannelUtil;
+import org.apache.pulsar.common.util.netty.NettyFutureUtil;
 import org.apache.pulsar.functions.utils.Exceptions;
 import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID;
 import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException;
@@ -236,6 +237,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
     private final long maxPendingBytesPerThread;
     private final long resumeThresholdPendingBytesPerThread;
 
+    private final long connectionLivenessCheckTimeoutMillis;
+
     // Number of bytes pending to be published from a single specific IO thread.
     private static final FastThreadLocal pendingBytesPerThread = new FastThreadLocal() {
         @Override
@@ -252,7 +255,6 @@ protected Set initialValue() throws Exception {
         }
     };
 
-
     enum State {
         Start, Connected, Failed, Connecting
     }
@@ -272,6 +274,8 @@ public ServerCnx(PulsarService pulsar, String listenerName) {
         this.state = State.Start;
         ServiceConfiguration conf = pulsar.getConfiguration();
 
+        this.connectionLivenessCheckTimeoutMillis = conf.getConnectionLivenessCheckTimeoutMillis();
+
         // This maps are not heavily contended since most accesses are within the cnx thread
         this.producers = ConcurrentLongHashMap.>newBuilder()
                 .expectedItems(8)
@@ -375,6 +379,11 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
         });
         this.topicListService.inactivate();
         this.service.getPulsarStats().recordConnectionClose();
+
+        // complete possible pending connection check future
+        if (connectionCheckInProgress != null && !connectionCheckInProgress.isDone()) {
+            connectionCheckInProgress.complete(false);
+        }
     }
 
     @Override
@@ -3325,6 +3334,43 @@ public String clientSourceAddress() {
         }
     }
 
+    CompletableFuture connectionCheckInProgress;
+
+    @Override
+    public CompletableFuture checkConnectionLiveness() {
+        if (connectionLivenessCheckTimeoutMillis > 0) {
+            return NettyFutureUtil.toCompletableFuture(ctx.executor().submit(() -> {
+                if (connectionCheckInProgress != null) {
+                    return connectionCheckInProgress;
+                } else {
+                    final CompletableFuture finalConnectionCheckInProgress = new CompletableFuture<>();
+                    connectionCheckInProgress = finalConnectionCheckInProgress;
+                    ctx.executor().schedule(() -> {
+                        if (finalConnectionCheckInProgress == connectionCheckInProgress
+                                && !finalConnectionCheckInProgress.isDone()) {
+                            log.warn("[{}] Connection check timed out. Closing connection.", remoteAddress);
+                            ctx.close();
+                        }
+                    }, connectionLivenessCheckTimeoutMillis, TimeUnit.MILLISECONDS);
+                    sendPing();
+                    return finalConnectionCheckInProgress;
+                }
+            })).thenCompose(java.util.function.Function.identity());
+        } else {
+            // check is disabled
+            return CompletableFuture.completedFuture((Boolean) null);
+        }
+    }
+
+    @Override
+    protected void messageReceived() {
+        super.messageReceived();
+        if (connectionCheckInProgress != null && !connectionCheckInProgress.isDone()) {
+            connectionCheckInProgress.complete(true);
+            connectionCheckInProgress = null;
+        }
+    }
+
     private static void logAuthException(SocketAddress remoteAddress, String operation,
                                          String principal, Optional topic, Throwable ex) {
         String topicString = topic.map(t -> ", topic=" + t.toString()).orElse("");
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java
index 511445da71dbe..e0ed75020bc82 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/StickyKeyConsumerSelector.java
@@ -20,7 +20,7 @@
 
 import java.util.List;
 import java.util.Map;
-import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException;
+import java.util.concurrent.CompletableFuture;
 import org.apache.pulsar.client.api.Range;
 import org.apache.pulsar.common.util.Murmur3_32Hash;
 
@@ -33,7 +33,7 @@ public interface StickyKeyConsumerSelector {
      *
      * @param consumer new consumer
      */
-    void addConsumer(Consumer consumer) throws ConsumerAssignException;
+    CompletableFuture addConsumer(Consumer consumer);
 
     /**
      * Remove the consumer.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
index 2abacf80ef3d2..f1aaca2b290fa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
@@ -21,6 +21,7 @@
 import io.netty.handler.codec.haproxy.HAProxyMessage;
 import io.netty.util.concurrent.Promise;
 import java.net.SocketAddress;
+import java.util.concurrent.CompletableFuture;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
 
 public interface TransportCnx {
@@ -78,4 +79,12 @@ public interface TransportCnx {
 
     String clientSourceAddress();
 
+    /***
+     * Check if the connection is still alive
+     * by actively sending a Ping message to the client.
+     *
+     * @return a completable future where the result is true if the connection is alive, false otherwise. The result
+     * is null if the connection liveness check is disabled.
+     */
+    CompletableFuture checkConnectionLiveness();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java
index d679106094178..c106b1603f6bd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java
@@ -35,6 +35,7 @@
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.stats.Rate;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -67,20 +68,21 @@ public NonPersistentDispatcherMultipleConsumers(NonPersistentTopic topic, Subscr
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
         if (IS_CLOSED_UPDATER.get(this) == TRUE) {
             log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer);
             consumer.disconnect();
-            return;
+            return CompletableFuture.completedFuture(null);
         }
 
         if (isConsumersExceededOnSubscription()) {
             log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit", name);
-            throw new ConsumerBusyException("Subscription reached max consumers limit");
+            return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit"));
         }
 
         consumerList.add(consumer);
         consumerSet.add(consumer);
+        return CompletableFuture.completedFuture(null);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java
index 23814bfdbe22e..2cad253f96ee2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java
@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.service.BrokerServiceException;
@@ -39,6 +40,7 @@
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
 import org.apache.pulsar.common.api.proto.KeySharedMode;
 import org.apache.pulsar.common.protocol.Commands;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -85,20 +87,24 @@ public NonPersistentStickyKeyDispatcherMultipleConsumers(NonPersistentTopic topi
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
         if (IS_CLOSED_UPDATER.get(this) == TRUE) {
             log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer);
             consumer.disconnect();
-            return;
-        }
-        super.addConsumer(consumer);
-        try {
-            selector.addConsumer(consumer);
-        } catch (BrokerServiceException e) {
-            consumerSet.removeAll(consumer);
-            consumerList.remove(consumer);
-            throw e;
+            return CompletableFuture.completedFuture(null);
         }
+        return super.addConsumer(consumer).thenCompose(__ ->
+                selector.addConsumer(consumer).handle((value, ex) -> {
+                    if (ex != null) {
+                        synchronized (NonPersistentStickyKeyDispatcherMultipleConsumers.this) {
+                            consumerSet.removeAll(consumer);
+                            consumerList.remove(consumer);
+                        }
+                        throw FutureUtil.wrapToCompletionException(ex);
+                    } else {
+                        return value;
+                    }
+                }));
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
index 3af886a528153..fc6df4a340997 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
@@ -172,12 +172,7 @@ public synchronized CompletableFuture addConsumer(Consumer consumer) {
             }
         }
 
-        try {
-            dispatcher.addConsumer(consumer);
-            return CompletableFuture.completedFuture(null);
-        } catch (BrokerServiceException brokerServiceException) {
-            return FutureUtil.failedFuture(brokerServiceException);
-        }
+        return dispatcher.addConsumer(consumer);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 7ff6e72d02aed..9540f6efe0005 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -75,6 +75,7 @@
 import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.Codec;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -153,11 +154,11 @@ public PersistentDispatcherMultipleConsumers(PersistentTopic topic, ManagedCurso
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
         if (IS_CLOSED_UPDATER.get(this) == TRUE) {
             log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer);
             consumer.disconnect();
-            return;
+            return CompletableFuture.completedFuture(null);
         }
         if (consumerList.isEmpty()) {
             if (havePendingRead || havePendingReplayRead) {
@@ -178,7 +179,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce
 
         if (isConsumersExceededOnSubscription()) {
             log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit", name);
-            throw new ConsumerBusyException("Subscription reached max consumers limit");
+            return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit"));
         }
 
         consumerList.add(consumer);
@@ -187,6 +188,8 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce
             consumerList.sort(Comparator.comparingInt(Consumer::getPriorityLevel));
         }
         consumerSet.add(consumer);
+
+        return CompletableFuture.completedFuture(null);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index 0c29421edd4ef..a0292729e734f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -31,6 +31,7 @@
 import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.bookkeeper.mledger.ManagedCursor;
@@ -52,6 +53,7 @@
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
 import org.apache.pulsar.common.api.proto.KeySharedMode;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -107,31 +109,37 @@ public StickyKeyConsumerSelector getSelector() {
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+    public synchronized CompletableFuture addConsumer(Consumer consumer) {
         if (IS_CLOSED_UPDATER.get(this) == TRUE) {
             log.warn("[{}] Dispatcher is already closed. Closing consumer {}", name, consumer);
             consumer.disconnect();
-            return;
-        }
-        super.addConsumer(consumer);
-        try {
-            selector.addConsumer(consumer);
-        } catch (BrokerServiceException e) {
-            consumerSet.removeAll(consumer);
-            consumerList.remove(consumer);
-            throw e;
-        }
-
-        PositionImpl readPositionWhenJoining = (PositionImpl) cursor.getReadPosition();
-        consumer.setReadPositionWhenJoining(readPositionWhenJoining);
-        // If this was the 1st consumer, or if all the messages are already acked, then we
-        // don't need to do anything special
-        if (!allowOutOfOrderDelivery
-                && recentlyJoinedConsumers != null
-                && consumerList.size() > 1
-                && cursor.getNumberOfEntriesSinceFirstNotAckedMessage() > 1) {
-            recentlyJoinedConsumers.put(consumer, readPositionWhenJoining);
+            return CompletableFuture.completedFuture(null);
         }
+        return super.addConsumer(consumer).thenCompose(__ ->
+                selector.addConsumer(consumer).handle((result, ex) -> {
+                    if (ex != null) {
+                        synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) {
+                            consumerSet.removeAll(consumer);
+                            consumerList.remove(consumer);
+                        }
+                        throw FutureUtil.wrapToCompletionException(ex);
+                    }
+                    return result;
+                })
+        ).thenRun(() -> {
+            synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) {
+                PositionImpl readPositionWhenJoining = (PositionImpl) cursor.getReadPosition();
+                consumer.setReadPositionWhenJoining(readPositionWhenJoining);
+                // If this was the 1st consumer, or if all the messages are already acked, then we
+                // don't need to do anything special
+                if (!allowOutOfOrderDelivery
+                        && recentlyJoinedConsumers != null
+                        && consumerList.size() > 1
+                        && cursor.getNumberOfEntriesSinceFirstNotAckedMessage() > 1) {
+                    recentlyJoinedConsumers.put(consumer, readPositionWhenJoining);
+                }
+            }
+        });
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 4ed191a9b4f61..d283cac77c7b6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -293,12 +293,7 @@ public CompletableFuture addConsumer(Consumer consumer) {
                     }
                 }
 
-                try {
-                    dispatcher.addConsumer(consumer);
-                    return CompletableFuture.completedFuture(null);
-                } catch (BrokerServiceException brokerServiceException) {
-                    return FutureUtil.failedFuture(brokerServiceException);
-                }
+                return dispatcher.addConsumer(consumer);
             }
         });
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java
index cc2ec3444d54d..332cccc2d2c6a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java
@@ -249,8 +249,8 @@ protected void reScheduleRead() {
         }
 
         @Override
-        public void addConsumer(Consumer consumer) throws BrokerServiceException {
-
+        public CompletableFuture addConsumer(Consumer consumer) {
+            return CompletableFuture.completedFuture(null);
         }
 
         @Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java
index e36c656e3faea..f3828981c8edd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/HashRangeExclusiveStickyKeyConsumerSelectorTest.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.pulsar.broker.service;
 
 import static org.mockito.Mockito.mock;
@@ -26,6 +27,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import org.apache.pulsar.client.api.Range;
 import org.apache.pulsar.common.api.proto.IntRange;
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
@@ -37,7 +40,7 @@
 public class HashRangeExclusiveStickyKeyConsumerSelectorTest {
 
     @Test
-    public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignException {
+    public void testConsumerSelect() throws ExecutionException, InterruptedException {
 
         HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
         Consumer consumer1 = mock(Consumer.class);
@@ -46,8 +49,8 @@ public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignExc
         keySharedMeta1.addHashRange().setStart(0).setEnd(2);
         when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
         Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
-        selector.addConsumer(consumer1);
-        Assert.assertEquals(selector.getRangeConsumer().size(),2);
+        selector.addConsumer(consumer1).get();
+        Assert.assertEquals(selector.getRangeConsumer().size(), 2);
         Consumer selectedConsumer;
         for (int i = 0; i < 3; i++) {
             selectedConsumer = selector.select(i);
@@ -62,8 +65,8 @@ public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignExc
         keySharedMeta2.addHashRange().setStart(3).setEnd(9);
         when(consumer2.getKeySharedMeta()).thenReturn(keySharedMeta2);
         Assert.assertEquals(consumer2.getKeySharedMeta(), keySharedMeta2);
-        selector.addConsumer(consumer2);
-        Assert.assertEquals(selector.getRangeConsumer().size(),4);
+        selector.addConsumer(consumer2).get();
+        Assert.assertEquals(selector.getRangeConsumer().size(), 4);
 
         for (int i = 3; i < 10; i++) {
             selectedConsumer = selector.select(i);
@@ -76,32 +79,46 @@ public void testConsumerSelect() throws BrokerServiceException.ConsumerAssignExc
         }
 
         selector.removeConsumer(consumer1);
-        Assert.assertEquals(selector.getRangeConsumer().size(),2);
+        Assert.assertEquals(selector.getRangeConsumer().size(), 2);
         selectedConsumer = selector.select(1);
         Assert.assertNull(selectedConsumer);
 
         selector.removeConsumer(consumer2);
-        Assert.assertEquals(selector.getRangeConsumer().size(),0);
+        Assert.assertEquals(selector.getRangeConsumer().size(), 0);
         selectedConsumer = selector.select(5);
         Assert.assertNull(selectedConsumer);
     }
 
-    @Test(expectedExceptions = BrokerServiceException.ConsumerAssignException.class)
-    public void testEmptyRanges() throws BrokerServiceException.ConsumerAssignException {
+    @Test
+    public void testEmptyRanges() {
         HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
         Consumer consumer = mock(Consumer.class);
         KeySharedMeta keySharedMeta = new KeySharedMeta()
                 .setKeySharedMode(KeySharedMode.STICKY);
         when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
-        selector.addConsumer(consumer);
+        try {
+            selector.addConsumer(consumer).get();
+            Assert.fail("Should have failed");
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerAssignException);
+        }
     }
 
-    @Test(expectedExceptions = BrokerServiceException.ConsumerAssignException.class)
-    public void testNullKeySharedMeta() throws BrokerServiceException.ConsumerAssignException {
+    @Test
+    public void testNullKeySharedMeta() throws ExecutionException, InterruptedException {
         HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
         Consumer consumer = mock(Consumer.class);
         when(consumer.getKeySharedMeta()).thenReturn(null);
-        selector.addConsumer(consumer);
+        try {
+            selector.addConsumer(consumer).get();
+            Assert.fail("Should have failed");
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerAssignException);
+        }
     }
 
     @Test(expectedExceptions = IllegalArgumentException.class)
@@ -110,7 +127,7 @@ public void testInvalidRangeTotal() {
     }
 
     @Test
-    public void testGetConsumerKeyHashRanges() throws BrokerServiceException.ConsumerAssignException {
+    public void testGetConsumerKeyHashRanges() throws ExecutionException, InterruptedException {
         HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
         List consumerName = Arrays.asList("consumer1", "consumer2", "consumer3", "consumer4");
         List range = Arrays.asList(new int[] {0, 2}, new int[] {3, 7}, new int[] {9, 12}, new int[] {15, 20});
@@ -125,7 +142,7 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume
             when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
             when(consumer.consumerName()).thenReturn(consumerName.get(index));
             Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
-            selector.addConsumer(consumer);
+            selector.addConsumer(consumer).get();
             consumers.add(consumer);
         }
 
@@ -157,7 +174,7 @@ public void testGetConsumerKeyHashRangesWithSameConsumerName() throws Exception
             when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
             when(consumer.consumerName()).thenReturn(consumerName);
             Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
-            selector.addConsumer(consumer);
+            selector.addConsumer(consumer).get();
             consumers.add(consumer);
         }
 
@@ -173,16 +190,19 @@ public void testGetConsumerKeyHashRangesWithSameConsumerName() throws Exception
     }
 
     @Test
-    public void testSingleRangeConflict() throws BrokerServiceException.ConsumerAssignException {
+    public void testSingleRangeConflict() throws ExecutionException, InterruptedException {
         HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
         Consumer consumer1 = mock(Consumer.class);
+        TransportCnx transportCnx = mock(TransportCnx.class);
+        when(consumer1.cnx()).thenReturn(transportCnx);
+        when(transportCnx.checkConnectionLiveness()).thenReturn(CompletableFuture.completedFuture(null));
         KeySharedMeta keySharedMeta1 = new KeySharedMeta()
                 .setKeySharedMode(KeySharedMode.STICKY);
         keySharedMeta1.addHashRange().setStart(2).setEnd(5);
         when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
         Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
-        selector.addConsumer(consumer1);
-        Assert.assertEquals(selector.getRangeConsumer().size(),2);
+        selector.addConsumer(consumer1).get();
+        Assert.assertEquals(selector.getRangeConsumer().size(), 2);
 
         final List testRanges = new ArrayList<>();
         testRanges.add(new IntRange().setStart(4).setEnd(6));
@@ -203,25 +223,29 @@ public void testSingleRangeConflict() throws BrokerServiceException.ConsumerAssi
             when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
             Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
             try {
-                selector.addConsumer(consumer);
+                selector.addConsumer(consumer).get();
                 Assert.fail("should be failed");
-            } catch (BrokerServiceException.ConsumerAssignException ignore) {
+            } catch (ExecutionException | InterruptedException e) {
+                // ignore
             }
-            Assert.assertEquals(selector.getRangeConsumer().size(),2);
+            Assert.assertEquals(selector.getRangeConsumer().size(), 2);
         }
     }
 
     @Test
-    public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAssignException {
+    public void testMultipleRangeConflict() throws ExecutionException, InterruptedException {
         HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
         Consumer consumer1 = mock(Consumer.class);
+        TransportCnx transportCnx = mock(TransportCnx.class);
+        when(consumer1.cnx()).thenReturn(transportCnx);
+        when(transportCnx.checkConnectionLiveness()).thenReturn(CompletableFuture.completedFuture(null));
         KeySharedMeta keySharedMeta1 = new KeySharedMeta()
                 .setKeySharedMode(KeySharedMode.STICKY);
         keySharedMeta1.addHashRange().setStart(2).setEnd(5);
         when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
         Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
-        selector.addConsumer(consumer1);
-        Assert.assertEquals(selector.getRangeConsumer().size(),2);
+        selector.addConsumer(consumer1).get();
+        Assert.assertEquals(selector.getRangeConsumer().size(), 2);
 
         final List> testRanges = new ArrayList<>();
         testRanges.add(List.of(
@@ -242,11 +266,12 @@ public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAs
             when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
             Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
             try {
-                selector.addConsumer(consumer);
+                selector.addConsumer(consumer).get();
                 Assert.fail("should be failed");
-            } catch (BrokerServiceException.ConsumerAssignException ignore) {
+            } catch (ExecutionException | InterruptedException e) {
+                // ignore
             }
-            Assert.assertEquals(selector.getRangeConsumer().size(),2);
+            Assert.assertEquals(selector.getRangeConsumer().size(), 2);
         }
     }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java
index b28f7a6028084..51cd61afd6362 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.common.protocol;
 
 import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
+import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.util.concurrent.ScheduledFuture;
 import java.net.SocketAddress;
@@ -56,7 +57,7 @@ public PulsarHandler(int keepAliveInterval, TimeUnit unit) {
     }
 
     @Override
-    protected final void messageReceived() {
+    protected void messageReceived() {
         waitingForPingResponse = false;
     }
 
@@ -120,14 +121,7 @@ private void handleKeepAliveTimeout() {
                 log.debug("[{}] Sending ping message", ctx.channel());
             }
             waitingForPingResponse = true;
-            ctx.writeAndFlush(Commands.newPing())
-                    .addListener(future -> {
-                        if (!future.isSuccess()) {
-                            log.warn("[{}] Forcing connection to close since cannot send a ping message.",
-                                    ctx.channel(), future.cause());
-                            ctx.close();
-                        }
-                    });
+            sendPing();
         } else {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Peer doesn't support keep-alive", ctx.channel());
@@ -135,6 +129,17 @@ private void handleKeepAliveTimeout() {
         }
     }
 
+    protected ChannelFuture sendPing() {
+        return ctx.writeAndFlush(Commands.newPing())
+                .addListener(future -> {
+                    if (!future.isSuccess()) {
+                        log.warn("[{}] Forcing connection to close since cannot send a ping message.",
+                                ctx.channel(), future.cause());
+                        ctx.close();
+                    }
+                });
+    }
+
     public void cancelKeepAliveTask() {
         if (keepAliveTask != null) {
             keepAliveTask.cancel(false);
diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml
index 03ec0aed8b56d..b9a68cba2c5e1 100644
--- a/pulsar-proxy/pom.xml
+++ b/pulsar-proxy/pom.xml
@@ -180,6 +180,11 @@
       ipaddress
       ${seancfoley.ipaddress.version}
     
+    
+      org.testcontainers
+      testcontainers
+      test
+    
   
   
     
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
index 6ec597ec1cfc3..b62b3bacf0114 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java
@@ -29,6 +29,7 @@
 import java.util.concurrent.Semaphore;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.impl.BinaryProtoLookupService;
 import org.apache.pulsar.common.api.proto.CommandGetSchema;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
 import org.apache.pulsar.common.api.proto.CommandLookupTopic;
@@ -155,7 +156,7 @@ private void performLookup(long clientRequestId, String topic, String brokerServ
                     writeAndFlush(
                         Commands.newLookupErrorResponse(getServerError(t), t.getMessage(), clientRequestId));
                 } else {
-                    String brokerUrl = connectWithTLS ? r.brokerUrlTls : r.brokerUrl;
+                    String brokerUrl = resolveBrokerUrlFromLookupDataResult(r);
                     if (r.redirect) {
                         // Need to try the lookup again on a different broker
                         performLookup(clientRequestId, topic, brokerUrl, r.authoritative, numberOfRetries - 1);
@@ -186,6 +187,10 @@ private void performLookup(long clientRequestId, String topic, String brokerServ
         });
     }
 
+    protected String resolveBrokerUrlFromLookupDataResult(BinaryProtoLookupService.LookupDataResult r) {
+        return connectWithTLS ? r.brokerUrlTls : r.brokerUrl;
+    }
+
     public void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata partitionMetadata) {
         PARTITIONS_METADATA_REQUESTS.inc();
         if (log.isDebugEnabled()) {
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 9530389b524b3..36b3b4c6038fb 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -374,7 +374,7 @@ private synchronized void completeConnect() throws PulsarClientException {
             // and we'll take care of just topics and
             // partitions metadata lookups
             state = State.ProxyLookupRequests;
-            lookupProxyHandler = new LookupProxyHandler(service, this);
+            lookupProxyHandler = service.newLookupProxyHandler(this);
             final ByteBuf msg = Commands.newConnected(protocolVersionToAdvertise, false);
             writeAndFlush(msg);
         }
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
index 4cca24f5f4892..a934b8b078426 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java
@@ -528,4 +528,8 @@ public synchronized void addPrometheusRawMetricsProvider(PrometheusRawMetricsPro
     }
 
     private static final Logger LOG = LoggerFactory.getLogger(ProxyService.class);
+
+    protected LookupProxyHandler newLookupProxyHandler(ProxyConnection proxyConnection) {
+        return new LookupProxyHandler(this, proxyConnection);
+    }
 }
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java
new file mode 100644
index 0000000000000..97279659af626
--- /dev/null
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pulsar.proxy.server;
+
+import static org.mockito.Mockito.doReturn;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.BrokerTestUtil;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.broker.authentication.AuthenticationService;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.KeySharedPolicy;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerAccessMode;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.Range;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.BinaryProtoLookupService;
+import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.Testcontainers;
+import org.testcontainers.containers.SocatContainer;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class ProxyStuckConnectionTest extends MockedPulsarServiceBaseTest {
+
+    private static final Logger log = LoggerFactory.getLogger(ProxyStuckConnectionTest.class);
+
+    private ProxyService proxyService;
+    private ProxyConfiguration proxyConfig;
+    private SocatContainer socatContainer;
+
+    private String brokerServiceUriSocat;
+    private volatile boolean useBrokerSocatProxy = true;
+
+    @Override
+    @BeforeMethod
+    protected void setup() throws Exception {
+        useBrokerSocatProxy = true;
+        internalSetup();
+
+        int brokerPort = pulsar.getBrokerService().getListenPort().get();
+        Testcontainers.exposeHostPorts(brokerPort);
+
+        socatContainer = new SocatContainer();
+        socatContainer.withTarget(brokerPort, "host.testcontainers.internal", brokerPort);
+        socatContainer.start();
+        brokerServiceUriSocat = "pulsar://" + socatContainer.getHost() + ":" + socatContainer.getMappedPort(brokerPort);
+
+        proxyConfig = new ProxyConfiguration();
+        proxyConfig.setServicePort(Optional.ofNullable(0));
+        proxyConfig.setBrokerProxyAllowedTargetPorts("*");
+        proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl());
+
+        startProxyService();
+        // use the same port for subsequent restarts
+        proxyConfig.setServicePort(proxyService.getListenPort());
+    }
+
+    private void startProxyService() throws Exception {
+        proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService(
+                PulsarConfigurationLoader.convertFrom(proxyConfig))) {
+            @Override
+            protected LookupProxyHandler newLookupProxyHandler(ProxyConnection proxyConnection) {
+                return new TestLookupProxyHandler(this, proxyConnection);
+            }
+        });
+        doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore();
+        doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore();
+        proxyService.start();
+    }
+
+    @Override
+    @AfterMethod(alwaysRun = true)
+    protected void cleanup() throws Exception {
+        internalCleanup();
+        if (proxyService != null) {
+            proxyService.close();
+        }
+        if (socatContainer != null) {
+            socatContainer.close();
+        }
+    }
+
+    private final class TestLookupProxyHandler extends LookupProxyHandler {
+        public TestLookupProxyHandler(ProxyService proxy, ProxyConnection proxyConnection) {
+            super(proxy, proxyConnection);
+        }
+
+        @Override
+        protected String resolveBrokerUrlFromLookupDataResult(BinaryProtoLookupService.LookupDataResult r) {
+            return useBrokerSocatProxy ? brokerServiceUriSocat : super.resolveBrokerUrlFromLookupDataResult(r);
+        }
+    }
+
+    @Test
+    public void testKeySharedStickyWithStuckConnection() throws Exception {
+        @Cleanup
+        PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl())
+                // keep alive is set to 2 seconds to detect the dead connection on the client side
+                // the main focus of the test is to verify that the broker and proxy doesn't get stuck forever
+                // when there's a hanging connection from the proxy to the broker and that it doesn't cause issues
+                // such as hash range conflicts
+                .keepAliveInterval(2, TimeUnit.SECONDS)
+                .build();
+        String topicName = BrokerTestUtil.newUniqueName("persistent://sample/test/local/test-topic");
+
+        @Cleanup
+        Consumer consumer = client.newConsumer()
+                .topic(topicName)
+                .subscriptionName("test-subscription")
+                .subscriptionType(SubscriptionType.Key_Shared)
+                .keySharedPolicy(KeySharedPolicy.stickyHashRange()
+                        .ranges(Range.of(0, 65535)))
+                .receiverQueueSize(2)
+                .isAckReceiptEnabled(true)
+                .subscribe();
+
+        Set messages = new HashSet<>();
+
+        try (Producer producer = client.newProducer()
+                .topic(topicName)
+                .accessMode(ProducerAccessMode.Shared)
+                .enableBatching(false)
+                .create()) {
+            for (int i = 0; i < 10; i++) {
+                String message = "test" + i;
+                producer.newMessage().value(message.getBytes())
+                        .key("A")
+                        .send();
+                messages.add(message);
+            }
+        }
+
+        int counter = 0;
+        while (true) {
+            counter++;
+            Message msg = consumer.receive(15, TimeUnit.SECONDS);
+            if (msg == null) {
+                break;
+            }
+            String msgString = new String(msg.getData());
+            log.info("Received message {}", msgString);
+            try {
+                consumer.acknowledge(msg);
+            } catch (PulsarClientException e) {
+                log.error("Failed to ack message {}", msgString, e);
+            }
+            messages.remove(msgString);
+            log.info("Remaining messages {}", messages.size());
+            if (messages.size() == 0) {
+                break;
+            }
+            if (counter == 2) {
+                log.info(
+                        "Pausing connection between proxy and broker and making further connections from proxy "
+                                + "directly to broker");
+                useBrokerSocatProxy = false;
+                socatContainer.getDockerClient().pauseContainerCmd(socatContainer.getContainerId()).exec();
+            }
+        }
+
+        Assert.assertEquals(messages.size(), 0);
+        Assert.assertEquals(consumer.receive(1, TimeUnit.MILLISECONDS), null);
+    }
+}

From 11751b7da2316516a2c18c11b3bd4011641b93f4 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 10 Apr 2023 17:44:41 -0500
Subject: [PATCH 164/404] [feat] PIP-257: Add AuthenticationProviderOpenID
 (#19849)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PIP: #19771

### Motivation

This is the primary PR for PIP 257 (#19771). It adds an OpenID Conenct `AuthenticationProvider` implementation. The implementation is intended to be compliant with the OpenID Specs defined here: https://openid.net/developers/specs/. We specifically implement the discovery and these two:

> * [OpenID Connect Core](https://openid.net/specs/openid-connect-core-1_0.html) – Defines the core OpenID Connect functionality: authentication built on top of OAuth 2.0 and the use of claims to communicate information about the End-User
> * [OpenID Connect Discovery](https://openid.net/specs/openid-connect-discovery-1_0.html) – Defines how clients dynamically discover information about OpenID Providers

### Modifications

* Add new module `pulsar-broker-auth-oidc`
* Add implementation that relies on auth0 client libraries to verify the signature and claims of the JWT
* Use async http client for all http requests
* Cache the provider metadata and the JWKS results
* Support different types of `FallbackDiscoveryMode`s, as documented in the code. Essentially, this setting allows users to more easily integrate with k8s. We need this coupling with kubernetes to deal with some of the nuances of the k8s implementation. Note that this part of the code is experimental and is subject to change as requirements and cloud provider implementations change. One important reason we use the k8s client is because the API Server requires special configuration for authentication and TLS. Since these do not appear to be generic requirements, the K8s client simplifies this integration. Here is a reference to the decision that requires authentication by default for getting OIDC info https://github.com/kubernetes/kubernetes/pull/80724. That discussion also indicate to me that this is an isolated design decision in k8s. If we find that authentication is a generic requirement, we should easily be able to expand the existing feature at a later time.
* Add metrics to help quantify success and failure. (I had thought I would add audit logging, but that is an independent feature that we can add to the Pulsar framework. It seems outside the scope of an Authentication Provider implementation to implement this feature.)

### Verifying this change

There are many new tests to cover this new implementation. Some of the tests are unit tests while others are integration tests that rely on Wire Mock to return the public key information.

### Documentation

- [x] `doc-required`

This feature will need new docs.

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/35
---
 distribution/server/pom.xml                   |   6 +
 .../server/src/assemble/LICENSE.bin.txt       |   3 +
 pom.xml                                       |   2 +
 pulsar-broker-auth-oidc/pom.xml               | 179 ++++++
 .../oidc/AuthenticationExceptionCode.java     |  38 ++
 .../oidc/AuthenticationProviderOpenID.java    | 493 ++++++++++++++++
 .../oidc/AuthenticationStateOpenID.java       |  96 ++++
 .../authentication/oidc/ConfigUtils.java      | 143 +++++
 .../oidc/FallbackDiscoveryMode.java           |  61 ++
 .../broker/authentication/oidc/JwksCache.java | 202 +++++++
 .../oidc/OpenIDProviderMetadata.java          |  53 ++
 .../oidc/OpenIDProviderMetadataCache.java     | 232 ++++++++
 .../authentication/oidc/package-info.java     |  19 +
 ...ticationProviderOpenIDIntegrationTest.java | 530 ++++++++++++++++++
 .../AuthenticationProviderOpenIDTest.java     | 387 +++++++++++++
 .../oidc/AuthenticationStateOpenIDTest.java   |  59 ++
 .../authentication/oidc/ConfigUtilsTest.java  | 151 +++++
 .../test/java/resources/fakeKubeConfig.yaml   |  37 ++
 18 files changed, 2691 insertions(+)
 create mode 100644 pulsar-broker-auth-oidc/pom.xml
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java
 create mode 100644 pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java
 create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
 create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java
 create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java
 create mode 100644 pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java
 create mode 100644 pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml

diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index 2043da516cf40..6b225bfc00c0b 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -46,6 +46,12 @@
       ${project.version}
     
 
+    
+      ${project.groupId}
+      pulsar-broker-auth-oidc
+      ${project.version}
+    
+
     
       ${project.groupId}
       pulsar-broker-auth-sasl
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 9205e072f7efe..4cb1d1544a63e 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -519,6 +519,9 @@ MIT License
     - org.checkerframework-checker-qual-3.12.0.jar
  * oshi
     - com.github.oshi-oshi-core-java11-6.4.0.jar
+ * Auth0, Inc.
+    - com.auth0-java-jwt-4.3.0.jar
+    - com.auth0-jwks-rsa-0.22.0.jar
 Protocol Buffers License
  * Protocol Buffers
    - com.google.protobuf-protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt
diff --git a/pom.xml b/pom.xml
index d6d7fdf44689d..389d9f6f83487 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2159,6 +2159,7 @@ flexible messaging model and an intuitive client API.
         pulsar-broker-auth-athenz
         pulsar-client-auth-athenz
         pulsar-sql
+        pulsar-broker-auth-oidc
         pulsar-broker-auth-sasl
         pulsar-client-auth-sasl
         pulsar-config-validation
@@ -2217,6 +2218,7 @@ flexible messaging model and an intuitive client API.
         pulsar-websocket
         pulsar-proxy
         pulsar-testclient
+        pulsar-broker-auth-oidc
         pulsar-broker-auth-sasl
         pulsar-client-auth-sasl
         pulsar-config-validation
diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml
new file mode 100644
index 0000000000000..9ad0363775a13
--- /dev/null
+++ b/pulsar-broker-auth-oidc/pom.xml
@@ -0,0 +1,179 @@
+
+
+
+  4.0.0
+  
+    org.apache.pulsar
+    pulsar
+    3.0.0-SNAPSHOT
+  
+
+  pulsar-broker-auth-oidc
+  jar
+  Open ID Connect authentication plugin for broker
+
+  
+    0.11.5
+  
+
+  
+
+    
+      ${project.groupId}
+      pulsar-broker-common
+      ${project.version}
+      
+        
+          io.grpc
+          *
+        
+      
+    
+
+    
+      com.auth0
+      java-jwt
+      4.3.0
+    
+
+    
+      com.auth0
+      jwks-rsa
+      0.22.0
+    
+
+    
+      com.github.ben-manes.caffeine
+      caffeine
+    
+
+    
+      org.asynchttpclient
+      async-http-client
+    
+
+    
+      io.kubernetes
+      client-java
+      ${kubernetesclient.version}
+      
+        
+        
+          io.prometheus
+          simpleclient_httpserver
+        
+      
+    
+
+    
+      io.jsonwebtoken
+      jjwt-api
+      ${jsonwebtoken.version}
+      test
+    
+    
+      io.jsonwebtoken
+      jjwt-impl
+      ${jsonwebtoken.version}
+      test
+    
+
+    
+      com.github.tomakehurst
+      wiremock-jre8
+      ${wiremock.version}
+      test
+    
+
+  
+
+  
+    
+      
+      test-jar-dependencies
+      
+        
+          maven.test.skip
+          !true
+        
+      
+      
+        
+          ${project.groupId}
+          pulsar-broker
+          ${project.version}
+          test
+          test-jar
+        
+      
+    
+  
+
+
+  
+    
+      
+        org.gaul
+        modernizer-maven-plugin
+        
+          true
+          8
+        
+        
+          
+            modernizer
+            verify
+            
+              modernizer
+            
+          
+        
+      
+
+      
+        org.apache.maven.plugins
+        maven-checkstyle-plugin
+        
+          
+            checkstyle
+            verify
+            
+              check
+            
+          
+        
+      
+      
+        org.apache.maven.plugins
+        maven-surefire-plugin
+        
+          
+            src/test/java/resources/fakeKubeConfig.yaml
+            ${project.basedir}/target/kubeconfig.yaml
+          
+        
+      
+    
+  
+
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java
new file mode 100644
index 0000000000000..5f89f5f1370f1
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationExceptionCode.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+/**
+ * Enum used to classify the types of exceptions encountered
+ * when attempting JWT verification.
+ */
+public enum AuthenticationExceptionCode {
+    UNSUPPORTED_ISSUER,
+    UNSUPPORTED_ALGORITHM,
+    ISSUER_MISMATCH,
+    ALGORITHM_MISMATCH,
+    INVALID_PUBLIC_KEY,
+    ERROR_RETRIEVING_PROVIDER_METADATA,
+    ERROR_RETRIEVING_PUBLIC_KEY,
+    ERROR_DECODING_JWT,
+    ERROR_VERIFYING_JWT,
+    ERROR_VERIFYING_JWT_SIGNATURE,
+    INVALID_JWT_CLAIM,
+    EXPIRED_JWT,
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
new file mode 100644
index 0000000000000..ae4774b6f6b6b
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsBoolean;
+import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt;
+import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsSet;
+import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsString;
+import com.auth0.jwk.InvalidPublicKeyException;
+import com.auth0.jwk.Jwk;
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.JWTVerifier;
+import com.auth0.jwt.RegisteredClaims;
+import com.auth0.jwt.algorithms.Algorithm;
+import com.auth0.jwt.exceptions.AlgorithmMismatchException;
+import com.auth0.jwt.exceptions.InvalidClaimException;
+import com.auth0.jwt.exceptions.JWTDecodeException;
+import com.auth0.jwt.exceptions.JWTVerificationException;
+import com.auth0.jwt.exceptions.SignatureVerificationException;
+import com.auth0.jwt.exceptions.TokenExpiredException;
+import com.auth0.jwt.interfaces.Claim;
+import com.auth0.jwt.interfaces.DecodedJWT;
+import com.auth0.jwt.interfaces.Verification;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.util.Config;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.net.SocketAddress;
+import java.security.PublicKey;
+import java.security.interfaces.ECPublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import javax.naming.AuthenticationException;
+import javax.net.ssl.SSLSession;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationProvider;
+import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
+import org.apache.pulsar.broker.authentication.AuthenticationState;
+import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics;
+import org.apache.pulsar.common.api.AuthData;
+import org.asynchttpclient.AsyncHttpClient;
+import org.asynchttpclient.AsyncHttpClientConfig;
+import org.asynchttpclient.DefaultAsyncHttpClient;
+import org.asynchttpclient.DefaultAsyncHttpClientConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An {@link AuthenticationProvider} implementation that supports the usage of a JSON Web Token (JWT)
+ * for client authentication. This implementation retrieves the PublicKey from the JWT issuer (assuming the
+ * issuer is in the configured allowed list) and then uses that Public Key to verify the validity of the JWT's
+ * signature.
+ *
+ * The Public Keys for a given provider are cached based on certain configured parameters to improve performance.
+ * The tradeoff here is that the longer Public Keys are cached, the longer an invalidated token could be used. One way
+ * to ensure caches are cleared is to restart all brokers.
+ *
+ * Class is called from multiple threads. The implementation must be thread safe. This class expects to be loaded once
+ * and then called concurrently for each new connection. The cache is backed by a GuavaCachedJwkProvider, which is
+ * thread-safe.
+ *
+ * Supported algorithms are: RS256, RS384, RS512, ES256, ES384, ES512 where the naming conventions follow
+ * this RFC: https://datatracker.ietf.org/doc/html/rfc7518#section-3.1.
+ */
+public class AuthenticationProviderOpenID implements AuthenticationProvider {
+    private static final Logger log = LoggerFactory.getLogger(AuthenticationProviderOpenID.class);
+
+    private static final String SIMPLE_NAME = AuthenticationProviderOpenID.class.getSimpleName();
+
+    // Must match the value used by the OAuth2 Client Plugin.
+    private static final String AUTH_METHOD_NAME = "token";
+
+    // This is backed by an ObjectMapper, which is thread safe. It is an optimization
+    // to share this for decoding JWTs for all connections to this broker.
+    private final JWT jwtLibrary = new JWT();
+
+    private Set issuers;
+
+    // This caches the map from Issuer URL to the jwks_uri served at the /.well-known/openid-configuration endpoint
+    private OpenIDProviderMetadataCache openIDProviderMetadataCache;
+
+    // A cache used to store the results of getting the JWKS from the jwks_uri for an issuer.
+    private JwksCache jwksCache;
+
+    private volatile AsyncHttpClient httpClient;
+
+    // A list of supported algorithms. This is the "alg" field on the JWT.
+    // Source for strings: https://datatracker.ietf.org/doc/html/rfc7518#section-3.1.
+    private static final String ALG_RS256 = "RS256";
+    private static final String ALG_RS384 = "RS384";
+    private static final String ALG_RS512 = "RS512";
+    private static final String ALG_ES256 = "ES256";
+    private static final String ALG_ES384 = "ES384";
+    private static final String ALG_ES512 = "ES512";
+
+    private long acceptedTimeLeewaySeconds;
+    private FallbackDiscoveryMode fallbackDiscoveryMode;
+    private String roleClaim = ROLE_CLAIM_DEFAULT;
+    private boolean isRoleClaimNotSubject;
+
+    static final String ALLOWED_TOKEN_ISSUERS = "openIDAllowedTokenIssuers";
+    static final String ISSUER_TRUST_CERTS_FILE_PATH = "openIDTokenIssuerTrustCertsFilePath";
+    static final String FALLBACK_DISCOVERY_MODE = "openIDFallbackDiscoveryMode";
+    static final String ALLOWED_AUDIENCES = "openIDAllowedAudiences";
+    static final String ROLE_CLAIM = "openIDRoleClaim";
+    static final String ROLE_CLAIM_DEFAULT = "sub";
+    static final String ACCEPTED_TIME_LEEWAY_SECONDS = "openIDAcceptedTimeLeewaySeconds";
+    static final int ACCEPTED_TIME_LEEWAY_SECONDS_DEFAULT = 0;
+    static final String CACHE_SIZE = "openIDCacheSize";
+    static final int CACHE_SIZE_DEFAULT = 5;
+    static final String CACHE_REFRESH_AFTER_WRITE_SECONDS = "openIDCacheRefreshAfterWriteSeconds";
+    static final int CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT = 18 * 60 * 60;
+    static final String CACHE_EXPIRATION_SECONDS = "openIDCacheExpirationSeconds";
+    static final int CACHE_EXPIRATION_SECONDS_DEFAULT = 24 * 60 * 60;
+    static final String HTTP_CONNECTION_TIMEOUT_MILLIS = "openIDHttpConnectionTimeoutMillis";
+    static final int HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT = 10_000;
+    static final String HTTP_READ_TIMEOUT_MILLIS = "openIDHttpReadTimeoutMillis";
+    static final int HTTP_READ_TIMEOUT_MILLIS_DEFAULT = 10_000;
+    static final String REQUIRE_HTTPS = "openIDRequireIssuersUseHttps";
+    static final boolean REQUIRE_HTTPS_DEFAULT = true;
+
+    // The list of audiences that are allowed to connect to this broker. A valid JWT must contain one of the audiences.
+    private String[] allowedAudiences;
+
+    @Override
+    public void initialize(ServiceConfiguration config) throws IOException {
+        this.allowedAudiences = validateAllowedAudiences(getConfigValueAsSet(config, ALLOWED_AUDIENCES));
+        this.roleClaim = getConfigValueAsString(config, ROLE_CLAIM, ROLE_CLAIM_DEFAULT);
+        this.isRoleClaimNotSubject = !ROLE_CLAIM_DEFAULT.equals(roleClaim);
+        this.acceptedTimeLeewaySeconds = getConfigValueAsInt(config, ACCEPTED_TIME_LEEWAY_SECONDS,
+                ACCEPTED_TIME_LEEWAY_SECONDS_DEFAULT);
+        boolean requireHttps = getConfigValueAsBoolean(config, REQUIRE_HTTPS, REQUIRE_HTTPS_DEFAULT);
+        this.fallbackDiscoveryMode = FallbackDiscoveryMode.valueOf(getConfigValueAsString(config,
+                FALLBACK_DISCOVERY_MODE, FallbackDiscoveryMode.DISABLED.name()));
+        this.issuers = validateIssuers(getConfigValueAsSet(config, ALLOWED_TOKEN_ISSUERS), requireHttps,
+                fallbackDiscoveryMode != FallbackDiscoveryMode.DISABLED);
+
+        int connectionTimeout = getConfigValueAsInt(config, HTTP_CONNECTION_TIMEOUT_MILLIS,
+                HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT);
+        int readTimeout = getConfigValueAsInt(config, HTTP_READ_TIMEOUT_MILLIS, HTTP_READ_TIMEOUT_MILLIS_DEFAULT);
+        String trustCertsFilePath = getConfigValueAsString(config, ISSUER_TRUST_CERTS_FILE_PATH, null);
+        SslContext sslContext = null;
+        if (trustCertsFilePath != null) {
+            // Use default settings for everything but the trust store.
+            sslContext = SslContextBuilder.forClient()
+                    .trustManager(new File(trustCertsFilePath))
+                    .build();
+        }
+        AsyncHttpClientConfig clientConfig = new DefaultAsyncHttpClientConfig.Builder()
+                .setConnectTimeout(connectionTimeout)
+                .setReadTimeout(readTimeout)
+                .setSslContext(sslContext)
+                .build();
+        httpClient = new DefaultAsyncHttpClient(clientConfig);
+        ApiClient k8sApiClient =
+                fallbackDiscoveryMode != FallbackDiscoveryMode.DISABLED ? Config.defaultClient() : null;
+        this.openIDProviderMetadataCache = new OpenIDProviderMetadataCache(config, httpClient, k8sApiClient);
+        this.jwksCache = new JwksCache(config, httpClient, k8sApiClient);
+    }
+
+    @Override
+    public String getAuthMethodName() {
+        return AUTH_METHOD_NAME;
+    }
+
+    /**
+     * Authenticate the parameterized {@link AuthenticationDataSource} by verifying the issuer is an allowed issuer,
+     * then retrieving the JWKS URI from the issuer, then retrieving the Public key from the JWKS URI, and finally
+     * verifying the JWT signature and claims.
+     *
+     * @param authData - the authData passed by the Pulsar Broker containing the token.
+     * @return the role, if the JWT is authenticated, otherwise a failed future.
+     */
+    @Override
+    public CompletableFuture authenticateAsync(AuthenticationDataSource authData) {
+        return authenticateTokenAsync(authData).thenApply(this::getRole);
+    }
+
+    /**
+     * Authenticate the parameterized {@link AuthenticationDataSource} and return the decoded JWT.
+     * @param authData - the authData containing the token.
+     * @return a completed future with the decoded JWT, if the JWT is authenticated. Otherwise, a failed future.
+     */
+    CompletableFuture authenticateTokenAsync(AuthenticationDataSource authData) {
+        String token;
+        try {
+            token = AuthenticationProviderToken.getToken(authData);
+        } catch (AuthenticationException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT);
+            return CompletableFuture.failedFuture(e);
+        }
+        return authenticateToken(token)
+                .whenComplete((jwt, e) -> {
+                    if (jwt != null) {
+                        AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName());
+                    }
+                    // Failure metrics are incremented within methods above
+                });
+    }
+
+    /**
+     * Get the role from a JWT at the configured role claim field.
+     * NOTE: does not do any verification of the JWT
+     * @param jwt - token to get the role from
+     * @return the role, or null, if it is not set on the JWT
+     */
+    String getRole(DecodedJWT jwt) {
+        try {
+            Claim roleClaim = jwt.getClaim(this.roleClaim);
+            if (roleClaim.isNull()) {
+                // The claim was not present in the JWT
+                return null;
+            }
+
+            String role = roleClaim.asString();
+            if (role != null) {
+                // The role is non null only if the JSON node is a text field
+                return role;
+            }
+
+            List roles = jwt.getClaim(this.roleClaim).asList(String.class);
+            if (roles == null || roles.size() == 0) {
+                return null;
+            } else if (roles.size() == 1) {
+                return roles.get(0);
+            } else {
+                log.debug("JWT for subject [{}] has multiple roles; using the first one.", jwt.getSubject());
+                return roles.get(0);
+            }
+        } catch (JWTDecodeException e) {
+            log.error("Exception while retrieving role from JWT", e);
+            return null;
+        }
+    }
+
+    /**
+     * Convert a JWT string into a {@link DecodedJWT}
+     * The benefit of using this method is that it utilizes the already instantiated {@link JWT} parser.
+     * WARNING: this method does not verify the authenticity of the token. It only decodes it.
+     *
+     * @param token - string JWT to be decoded
+     * @return a decoded JWT
+     * @throws AuthenticationException if the token string is null or if any part of the token contains
+     *         an invalid jwt or JSON format of each of the jwt parts.
+     */
+    DecodedJWT decodeJWT(String token) throws AuthenticationException {
+        if (token == null) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT);
+            throw new AuthenticationException("Invalid token: cannot be null");
+        }
+        try {
+            return jwtLibrary.decodeJwt(token);
+        } catch (JWTDecodeException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT);
+            throw new AuthenticationException("Unable to decode JWT: " + e.getMessage());
+        }
+    }
+
+    /**
+     * Authenticate the parameterized JWT.
+     *
+     * @param token - a nonnull JWT to authenticate
+     * @return a fully authenticated JWT, or AuthenticationException if the JWT is proven to be invalid in any way
+     */
+    private CompletableFuture authenticateToken(String token) {
+        if (token == null) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT);
+            return CompletableFuture.failedFuture(new AuthenticationException("JWT cannot be null"));
+        }
+        final DecodedJWT jwt;
+        try {
+            jwt = decodeJWT(token);
+        } catch (AuthenticationException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT);
+            return CompletableFuture.failedFuture(e);
+        }
+        return verifyIssuerAndGetJwk(jwt)
+                .thenCompose(jwk -> {
+                    try {
+                        if (!jwt.getAlgorithm().equals(jwk.getAlgorithm())) {
+                            incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH);
+                            return CompletableFuture.failedFuture(
+                                    new AuthenticationException("JWK's alg [" + jwk.getAlgorithm()
+                                            + "] does not match JWT's alg [" + jwt.getAlgorithm() + "]"));
+                        }
+                        // Verify the JWT signature
+                        // Throws exception if any verification check fails
+                        return CompletableFuture
+                                .completedFuture(verifyJWT(jwk.getPublicKey(), jwt.getAlgorithm(), jwt));
+                    } catch (InvalidPublicKeyException e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.INVALID_PUBLIC_KEY);
+                        return CompletableFuture.failedFuture(
+                                new AuthenticationException("Invalid public key: " + e.getMessage()));
+                    } catch (AuthenticationException e) {
+                        return CompletableFuture.failedFuture(e);
+                    }
+                });
+    }
+
+    /**
+     * Verify the JWT's issuer (iss) claim is one of the allowed issuers and then retrieve the JWK from the issuer. If
+     * not, see {@link FallbackDiscoveryMode} for the fallback behavior.
+     * @param jwt - the token to use to discover the issuer's JWKS URI, which is then used to retrieve the issuer's
+     *            current public keys.
+     * @return a JWK that can be used to verify the JWT's signature
+     */
+    private CompletableFuture verifyIssuerAndGetJwk(DecodedJWT jwt) {
+        if (jwt.getIssuer() == null) {
+            incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER);
+            return CompletableFuture.failedFuture(new AuthenticationException("Issuer cannot be null"));
+        } else if (this.issuers.contains(jwt.getIssuer())) {
+            // Retrieve the metadata: https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
+            return openIDProviderMetadataCache.getOpenIDProviderMetadataForIssuer(jwt.getIssuer())
+                    .thenCompose(metadata -> jwksCache.getJwk(metadata.getJwksUri(), jwt.getKeyId()));
+        } else if (fallbackDiscoveryMode == FallbackDiscoveryMode.KUBERNETES_DISCOVER_TRUSTED_ISSUER) {
+            return openIDProviderMetadataCache.getOpenIDProviderMetadataForKubernetesApiServer(jwt.getIssuer())
+                    .thenCompose(metadata ->
+                            openIDProviderMetadataCache.getOpenIDProviderMetadataForIssuer(metadata.getIssuer()))
+                    .thenCompose(metadata -> jwksCache.getJwk(metadata.getJwksUri(), jwt.getKeyId()));
+        } else if (fallbackDiscoveryMode == FallbackDiscoveryMode.KUBERNETES_DISCOVER_PUBLIC_KEYS) {
+            return openIDProviderMetadataCache.getOpenIDProviderMetadataForKubernetesApiServer(jwt.getIssuer())
+                    .thenCompose(__ -> jwksCache.getJwkFromKubernetesApiServer(jwt.getKeyId()));
+        } else {
+            incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER);
+            return CompletableFuture
+                    .failedFuture(new AuthenticationException("Issuer not allowed: " + jwt.getIssuer()));
+        }
+    }
+
+    @Override
+    public AuthenticationState newAuthState(AuthData authData, SocketAddress remoteAddress, SSLSession sslSession)
+            throws AuthenticationException {
+        return new AuthenticationStateOpenID(this, remoteAddress, sslSession);
+    }
+
+    @Override
+    public void close() throws IOException {
+        httpClient.close();
+    }
+
+    /**
+     * Build and return a validator for the parameters.
+     *
+     * @param publicKey - the public key to use when configuring the validator
+     * @param publicKeyAlg - the algorithm for the parameterized public key
+     * @param jwt - jwt to be verified and returned (only if verified)
+     * @return a validator to use for validating a JWT associated with the parameterized public key.
+     * @throws AuthenticationException if the Public Key's algorithm is not supported or if the algorithm param does not
+     * match the Public Key's actual algorithm.
+     */
+    DecodedJWT verifyJWT(PublicKey publicKey,
+                                String publicKeyAlg,
+                                DecodedJWT jwt) throws AuthenticationException {
+        if (publicKeyAlg == null) {
+            incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ALGORITHM);
+            throw new AuthenticationException("PublicKey algorithm cannot be null");
+        }
+
+        Algorithm alg;
+        try {
+            switch (publicKeyAlg) {
+                case ALG_RS256:
+                    alg = Algorithm.RSA256((RSAPublicKey) publicKey, null);
+                    break;
+                case ALG_RS384:
+                    alg = Algorithm.RSA384((RSAPublicKey) publicKey, null);
+                    break;
+                case ALG_RS512:
+                    alg = Algorithm.RSA512((RSAPublicKey) publicKey, null);
+                    break;
+                case ALG_ES256:
+                    alg = Algorithm.ECDSA256((ECPublicKey) publicKey, null);
+                    break;
+                case ALG_ES384:
+                    alg = Algorithm.ECDSA384((ECPublicKey) publicKey, null);
+                    break;
+                case ALG_ES512:
+                    alg = Algorithm.ECDSA512((ECPublicKey) publicKey, null);
+                    break;
+                default:
+                    incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ALGORITHM);
+                    throw new AuthenticationException("Unsupported algorithm: " + publicKeyAlg);
+            }
+        } catch (ClassCastException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH);
+            throw new AuthenticationException("Expected PublicKey alg [" + publicKeyAlg + "] does match actual alg.");
+        }
+
+        // We verify issuer when retrieving the PublicKey, so it is not verified here.
+        // The claim presence requirements are based on https://openid.net/specs/openid-connect-basic-1_0.html#IDToken
+         Verification verifierBuilder = JWT.require(alg)
+                .acceptLeeway(acceptedTimeLeewaySeconds)
+                .withAnyOfAudience(allowedAudiences)
+                .withClaimPresence(RegisteredClaims.ISSUED_AT)
+                .withClaimPresence(RegisteredClaims.EXPIRES_AT)
+                .withClaimPresence(RegisteredClaims.NOT_BEFORE)
+                .withClaimPresence(RegisteredClaims.SUBJECT);
+
+        if (isRoleClaimNotSubject) {
+            verifierBuilder = verifierBuilder.withClaimPresence(roleClaim);
+        }
+
+        JWTVerifier verifier = verifierBuilder.build();
+
+        try {
+            return verifier.verify(jwt);
+        } catch (TokenExpiredException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.EXPIRED_JWT);
+            throw new AuthenticationException("JWT expired: " + e.getMessage());
+        } catch (SignatureVerificationException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_VERIFYING_JWT_SIGNATURE);
+            throw new AuthenticationException("JWT signature verification exception: " + e.getMessage());
+        } catch (InvalidClaimException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.INVALID_JWT_CLAIM);
+            throw new AuthenticationException("JWT contains invalid claim: " + e.getMessage());
+        } catch (AlgorithmMismatchException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH);
+            throw new AuthenticationException("JWT algorithm does not match Public Key algorithm: " + e.getMessage());
+        } catch (JWTDecodeException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_DECODING_JWT);
+            throw new AuthenticationException("Error while decoding JWT: " + e.getMessage());
+        } catch (JWTVerificationException | IllegalArgumentException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_VERIFYING_JWT);
+            throw new AuthenticationException("JWT verification failed: " + e.getMessage());
+        }
+    }
+
+    static void incrementFailureMetric(AuthenticationExceptionCode code) {
+        AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, "token", code.toString());
+    }
+
+    /**
+     * Validate the configured allow list of allowedIssuers. The allowedIssuers set must be nonempty in order for
+     * the plugin to authenticate any token. Thus, it fails initialization if the configuration is
+     * missing. Each issuer URL should use the HTTPS scheme. The plugin fails initialization if any
+     * issuer url is insecure, unless requireHttps is false.
+     * @param allowedIssuers - issuers to validate
+     * @param requireHttps - whether to require https for issuers.
+     * @param allowEmptyIssuers - whether to allow empty issuers. This setting only makes sense when kubernetes is used
+     *                   as a fallback issuer.
+     * @return the validated issuers
+     * @throws IllegalArgumentException if the allowedIssuers is empty, or contains insecure issuers when required
+     */
+    private Set validateIssuers(Set allowedIssuers, boolean requireHttps, boolean allowEmptyIssuers) {
+        if (allowedIssuers == null || (allowedIssuers.isEmpty() && !allowEmptyIssuers)) {
+            throw new IllegalArgumentException("Missing configured value for: " + ALLOWED_TOKEN_ISSUERS);
+        }
+        for (String issuer : allowedIssuers) {
+            if (!issuer.toLowerCase().startsWith("https://")) {
+                log.warn("Allowed issuer is not using https scheme: {}", issuer);
+                if (requireHttps) {
+                    throw new IllegalArgumentException("Issuer URL does not use https, but must: " + issuer);
+                }
+            }
+        }
+        return allowedIssuers;
+    }
+
+    /**
+     * Validate the configured allow list of allowedAudiences. The allowedAudiences must be set because
+     * JWT must have an audience claim.
+     * See https://openid.net/specs/openid-connect-basic-1_0.html#IDTokenValidation.
+     * @param allowedAudiences
+     * @return the validated audiences
+     */
+    String[] validateAllowedAudiences(Set allowedAudiences) {
+        if (allowedAudiences == null || allowedAudiences.isEmpty()) {
+            throw new IllegalArgumentException("Missing configured value for: " + ALLOWED_AUDIENCES);
+        }
+        return allowedAudiences.toArray(new String[0]);
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java
new file mode 100644
index 0000000000000..3046a6dd0e3b4
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenID.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import java.net.SocketAddress;
+import java.util.concurrent.CompletableFuture;
+import javax.naming.AuthenticationException;
+import javax.net.ssl.SSLSession;
+import org.apache.pulsar.broker.authentication.AuthenticationDataCommand;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationState;
+import org.apache.pulsar.common.api.AuthData;
+
+/**
+ * Class representing the authentication state of a single connection.
+ */
+class AuthenticationStateOpenID implements AuthenticationState {
+    private final AuthenticationProviderOpenID provider;
+    private AuthenticationDataSource authenticationDataSource;
+    private volatile String role;
+    private final SocketAddress remoteAddress;
+    private final SSLSession sslSession;
+    private volatile long expiration;
+
+    AuthenticationStateOpenID(
+            AuthenticationProviderOpenID provider,
+            SocketAddress remoteAddress,
+            SSLSession sslSession) {
+        this.provider = provider;
+        this.remoteAddress = remoteAddress;
+        this.sslSession = sslSession;
+    }
+
+    @Override
+    public String getAuthRole() throws AuthenticationException {
+        if (role == null) {
+            throw new AuthenticationException("Authentication has not completed");
+        }
+        return role;
+    }
+
+    @Deprecated
+    @Override
+    public AuthData authenticate(AuthData authData) throws AuthenticationException {
+        // This method is not expected to be called and is subject to removal.
+        throw new AuthenticationException("Not supported");
+    }
+
+    @Override
+    public CompletableFuture authenticateAsync(AuthData authData) {
+        final String token = new String(authData.getBytes(), UTF_8);
+        this.authenticationDataSource = new AuthenticationDataCommand(token, remoteAddress, sslSession);
+        return provider
+                .authenticateTokenAsync(authenticationDataSource)
+                .thenApply(jwt -> {
+                    this.role = provider.getRole(jwt);
+                    // OIDC requires setting the exp claim, so this should never be null.
+                    // We verify it is not null during token validation.
+                    this.expiration = jwt.getExpiresAt().getTime();
+                    // Single stage authentication, so return null here
+                    return null;
+                });
+    }
+
+    @Override
+    public AuthenticationDataSource getAuthDataSource() {
+        return authenticationDataSource;
+    }
+
+    @Override
+    public boolean isComplete() {
+        return role != null;
+    }
+
+    @Override
+    public boolean isExpired() {
+        return System.currentTimeMillis() > expiration;
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java
new file mode 100644
index 0000000000000..f62bf9c818653
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtils.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class ConfigUtils {
+    private static final Logger log = LoggerFactory.getLogger(ConfigUtils.class);
+
+    /**
+     * Get configured property as a string. If not configured, return null.
+     * @param conf - the configuration map
+     * @param configProp - the property to get
+     * @return a string from the conf or null, if the configuration property was not set
+     */
+    static String getConfigValueAsString(ServiceConfiguration conf,
+                                                String configProp) throws IllegalArgumentException {
+        String value = getConfigValueAsStringImpl(conf, configProp);
+        log.info("Configuration for [{}] is [{}]", configProp, value);
+        return value;
+    }
+
+    /**
+     * Get configured property as a string. If not configured, return null.
+     * @param conf - the configuration map
+     * @param configProp - the property to get
+     * @param defaultValue - the value to use if the configuration value is not set
+     * @return a string from the conf or the default value
+     */
+    static String getConfigValueAsString(ServiceConfiguration conf, String configProp,
+                                                String defaultValue) throws IllegalArgumentException {
+        String value = getConfigValueAsStringImpl(conf, configProp);
+        if (value == null) {
+            value = defaultValue;
+        }
+        log.info("Configuration for [{}] is [{}]", configProp, value);
+        return value;
+    }
+
+    /**
+     * Get configured property as a set. Split using a comma delimiter and remove any extra whitespace surrounding
+     * the commas. If not configured, return the empty set.
+     *
+     * @param conf - the map of configuration properties
+     * @param configProp - the property (key) to get
+     * @return a set of strings from the conf
+     */
+    static Set getConfigValueAsSet(ServiceConfiguration conf, String configProp) {
+        String value = getConfigValueAsStringImpl(conf, configProp);
+        if (StringUtils.isBlank(value)) {
+            log.info("Configuration for [{}] is the empty set.", configProp);
+            return Collections.emptySet();
+        }
+        Set set = Arrays.stream(value.trim().split("\\s*,\\s*")).collect(Collectors.toSet());
+        log.info("Configuration for [{}] is [{}].", configProp, String.join(", ", set));
+        return set;
+    }
+
+    private static String getConfigValueAsStringImpl(ServiceConfiguration conf,
+                                                     String configProp) throws IllegalArgumentException {
+        Object value = conf.getProperty(configProp);
+        if (value instanceof String) {
+            return (String) value;
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Utility method to get an integer from the {@link ServiceConfiguration}. If the value is not a valid long or the
+     * key is not present in the conf, the default value will be used.
+     *
+     * @param conf - the map of configuration properties
+     * @param configProp - the property (key) to get
+     * @param defaultValue - the value to use if the property is missing from the conf
+     * @return a long
+     */
+    static int getConfigValueAsInt(ServiceConfiguration conf, String configProp, int defaultValue) {
+        Object value = conf.getProperty(configProp);
+        if (value instanceof Integer) {
+            log.info("Configuration for [{}] is [{}]", configProp, value);
+            return (Integer) value;
+        } else if (value instanceof String) {
+            try {
+                return Integer.parseInt((String) value);
+            } catch (NumberFormatException numberFormatException) {
+                log.error("Expected configuration for [{}] to be a long, but got [{}]. Using default value: [{}]",
+                        configProp, value, defaultValue, numberFormatException);
+                return defaultValue;
+            }
+        } else {
+            log.info("Configuration for [{}] is using the default value: [{}]", configProp, defaultValue);
+            return defaultValue;
+        }
+    }
+
+    /**
+     * Utility method to get a boolean from the {@link ServiceConfiguration}. If the key is present in the conf,
+     * return the default value. If key is present the value is not a valid boolean, the result will be false.
+     *
+     * @param conf - the map of configuration properties
+     * @param configProp - the property (key) to get
+     * @param defaultValue - the value to use if the property is missing from the conf
+     * @return a boolean
+     */
+    static boolean getConfigValueAsBoolean(ServiceConfiguration conf, String configProp, boolean defaultValue) {
+        Object value = conf.getProperty(configProp);
+        if (value instanceof Boolean) {
+            log.info("Configuration for [{}] is [{}]", configProp, value);
+            return (boolean) value;
+        } else if (value instanceof String) {
+            boolean result = Boolean.parseBoolean((String) value);
+            log.info("Configuration for [{}] is [{}]", configProp, result);
+            return result;
+        } else {
+            log.info("Configuration for [{}] is using the default value: [{}]", configProp, defaultValue);
+            return defaultValue;
+        }
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java
new file mode 100644
index 0000000000000..5bf0c1b23fce6
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/FallbackDiscoveryMode.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import org.apache.pulsar.common.classification.InterfaceStability;
+
+/**
+ * These are the modes available for configuring how the Open ID Connect Authentication Provider should handle a JWT
+ * that has an issuer that is not explicitly in the allowed issuers set configured by
+ * {@link AuthenticationProviderOpenID#ALLOWED_TOKEN_ISSUERS}. The current implementations rely on using the Kubernetes
+ * Api Server's Open ID Connect features to discover an additional issuer or additional public keys to trust. See the
+ * Kubernetes documentation for more information on how Service Accounts can integrate with Open ID Connect.
+ * https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-issuer-discovery
+ */
+@InterfaceStability.Evolving
+public enum FallbackDiscoveryMode {
+    /**
+     * There will be no discovery of additional trusted issuers or public keys. This setting requires that operators
+     * explicitly allow all issuers that will be trusted. For the Kubernetes Service Account Token Projections to work,
+     * the operator must explicitly trust the issuer on the token's "iss" claim. This is the default setting because it
+     * is the only mode that explicitly follows the OIDC spec for verification of discovered provider configuration.
+     */
+    DISABLED,
+
+    /**
+     * The Kubernetes Api Server will be used to discover an additional trusted issuer by getting the issuer at the
+     * Api Server's /.well-known/openid-configuration endpoint, verifying that issuer matches the "iss" claim on the
+     * supplied token, then treating that issuer as a trusted issuer by discovering the jwks_uri via that issuer's
+     * /.well-known/openid-configuration endpoint. This mode can be helpful in EKS environments where the Api Server's
+     * public keys served at the /openid/v1/jwks endpoint are not the same as the public keys served at the issuer's
+     * jwks_uri. It fails to be OIDC compliant because the URL used to discover the provider configuration is not the
+     * same as the issuer claim on the token.
+     */
+    KUBERNETES_DISCOVER_TRUSTED_ISSUER,
+
+    /**
+     * The Kubernetes Api Server will be used to discover an additional set of valid public keys by getting the issuer
+     * at the Api Server's /.well-known/openid-configuration endpoint, verifying that issuer matches the "iss" claim on
+     * the supplied token, then calling the Api Server endpoint to get the public keys using a kubernetes client. This
+     * mode is currently useful getting the public keys from the Api Server because the Api Server requires custom TLS
+     * and authentication, and the kubernetes client automatically handles those. It fails to be OIDC compliant because
+     * the URL used to discover the provider configuration is not the same as the issuer claim on the token.
+     */
+    KUBERNETES_DISCOVER_PUBLIC_KEYS,
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
new file mode 100644
index 0000000000000..12ea7ec6b906d
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric;
+import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt;
+import com.auth0.jwk.Jwk;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.github.benmanes.caffeine.cache.AsyncCacheLoader;
+import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import io.kubernetes.client.openapi.ApiCallback;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.OpenidApi;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import javax.naming.AuthenticationException;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.asynchttpclient.AsyncHttpClient;
+
+public class JwksCache {
+
+    // Map from an issuer's JWKS URI to its JWKS. When the Issuer is not empty, use the fallback client.
+    private final AsyncLoadingCache, List> cache;
+
+    private final ObjectReader reader = new ObjectMapper().readerFor(HashMap.class);
+    private final AsyncHttpClient httpClient;
+    private final OpenidApi openidApi;
+
+    JwksCache(ServiceConfiguration config, AsyncHttpClient httpClient, ApiClient apiClient) throws IOException {
+        // Store the clients
+        this.httpClient = httpClient;
+        this.openidApi = apiClient != null ? new OpenidApi(apiClient) : null;
+
+        // Configure the cache
+        int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT);
+        int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS,
+                CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT);
+        int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS,
+                CACHE_EXPIRATION_SECONDS_DEFAULT);
+        AsyncCacheLoader, List> loader = (jwksUri, executor) -> {
+            if (jwksUri.isPresent()) {
+                return getJwksFromJwksUri(jwksUri.get());
+            } else {
+                return getJwksFromKubernetesApiServer();
+            }
+        };
+        this.cache = Caffeine.newBuilder()
+                .maximumSize(maxSize)
+                .refreshAfterWrite(refreshAfterWriteSeconds, TimeUnit.SECONDS)
+                .expireAfterWrite(expireAfterSeconds, TimeUnit.SECONDS)
+                .buildAsync(loader);
+    }
+
+    CompletableFuture getJwk(String jwksUri, String keyId) {
+        if (jwksUri == null) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+            return CompletableFuture.failedFuture(new IllegalArgumentException("jwksUri must not be null."));
+        }
+        return cache.get(Optional.of(jwksUri)).thenApply(jwks -> getJwkForKID(jwks, keyId));
+    }
+
+    private CompletableFuture> getJwksFromJwksUri(String jwksUri) {
+        return httpClient
+                .prepareGet(jwksUri)
+                .execute()
+                .toCompletableFuture()
+                .thenCompose(result -> {
+                    CompletableFuture> future = new CompletableFuture<>();
+                    try {
+                        HashMap jwks =
+                                reader.readValue(result.getResponseBodyAsBytes());
+                        future.complete(convertToJwks(jwksUri, jwks));
+                    } catch (AuthenticationException e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+                        future.completeExceptionally(e);
+                    } catch (Exception e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+                        future.completeExceptionally(new AuthenticationException(
+                                "Error retrieving public key at " + jwksUri + ": " + e.getMessage()));
+                    }
+                    return future;
+                });
+    }
+
+    CompletableFuture getJwkFromKubernetesApiServer(String keyId) {
+        if (openidApi == null) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+            return CompletableFuture.failedFuture(new AuthenticationException(
+                    "Failed to retrieve public key from Kubernetes API server: Kubernetes fallback is not enabled."));
+        }
+        return cache.get(Optional.empty(), (__, executor) -> getJwksFromKubernetesApiServer())
+                .thenApply(jwks -> getJwkForKID(jwks, keyId));
+    }
+
+    private CompletableFuture> getJwksFromKubernetesApiServer() {
+        CompletableFuture> future = new CompletableFuture<>();
+        try {
+            openidApi.getServiceAccountIssuerOpenIDKeysetAsync(new ApiCallback() {
+                @Override
+                public void onFailure(ApiException e, int statusCode, Map> responseHeaders) {
+                    incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+                    future.completeExceptionally(
+                            new AuthenticationException("Failed to retrieve public key from Kubernetes API server: "
+                                    + e.getMessage()));
+                }
+
+                @Override
+                public void onSuccess(String result, int statusCode, Map> responseHeaders) {
+                    try {
+                        HashMap jwks = reader.readValue(result);
+                        future.complete(convertToJwks("Kubernetes API server", jwks));
+                    } catch (AuthenticationException e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+                        future.completeExceptionally(e);
+                    } catch (Exception e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+                        future.completeExceptionally(new AuthenticationException(
+                                "Error retrieving public key at Kubernetes API server: " + e.getMessage()));
+                    }
+                }
+
+                @Override
+                public void onUploadProgress(long bytesWritten, long contentLength, boolean done) {
+
+                }
+
+                @Override
+                public void onDownloadProgress(long bytesRead, long contentLength, boolean done) {
+
+                }
+            });
+        } catch (ApiException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+            future.completeExceptionally(
+                    new AuthenticationException("Failed to retrieve public key from Kubernetes API server: "
+                            + e.getMessage()));
+        }
+        return future;
+    }
+
+    private Jwk getJwkForKID(List jwks, String keyId) {
+        for (Jwk jwk : jwks) {
+            if (jwk.getId().equals(keyId)) {
+                return jwk;
+            }
+        }
+        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+        throw new IllegalArgumentException("No JWK found for Key ID " + keyId);
+    }
+
+    /**
+     * The JWK Set is stored in the "keys" key see https://www.rfc-editor.org/rfc/rfc7517#section-5.1.
+     *
+     * @param jwksUri - the URI used to retrieve the JWKS
+     * @param jwks - the JWKS to convert
+     * @return a list of {@link Jwk}
+     */
+    private List convertToJwks(String jwksUri, Map jwks) throws AuthenticationException {
+        try {
+            @SuppressWarnings("unchecked")
+            List> jwkList = (List>) jwks.get("keys");
+            final List result = new ArrayList<>();
+            for (Map jwk : jwkList) {
+                result.add(Jwk.fromValues(jwk));
+            }
+            return result;
+        } catch (ClassCastException e) {
+            throw new AuthenticationException("Malformed JWKS returned by: " + jwksUri);
+        }
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java
new file mode 100644
index 0000000000000..553b3b882dbee
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadata.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * A Simple Class representing the essential fields of the OpenID Provider Metadata.
+ * Spec: https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
+ * Note that this class is only used for deserializing the JSON metadata response from
+ * calling a provider's /.well-known/openid-configuration endpoint.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OpenIDProviderMetadata {
+
+    private final String issuer;
+    private final String jwksUri;
+
+    @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+    public OpenIDProviderMetadata(@JsonProperty("issuer") String issuer, @JsonProperty("jwks_uri") String jwksUri) {
+        this.issuer = issuer;
+        this.jwksUri = jwksUri;
+    }
+
+    @JsonGetter
+    public String getIssuer() {
+        return issuer;
+    }
+
+    @JsonGetter
+    public String getJwksUri() {
+        return jwksUri;
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java
new file mode 100644
index 0000000000000..33d11f35a349a
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric;
+import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.github.benmanes.caffeine.cache.AsyncCacheLoader;
+import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import io.kubernetes.client.openapi.ApiCallback;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.WellKnownApi;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+import javax.naming.AuthenticationException;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.asynchttpclient.AsyncHttpClient;
+
+/**
+ * Class used to cache metadata responses from OpenID Providers.
+ */
+class OpenIDProviderMetadataCache {
+
+    private final ObjectReader reader = new ObjectMapper().readerFor(OpenIDProviderMetadata.class);
+    private final AsyncHttpClient httpClient;
+    private final WellKnownApi wellKnownApi;
+    private final AsyncLoadingCache, OpenIDProviderMetadata> cache;
+    private static final String WELL_KNOWN_OPENID_CONFIG = ".well-known/openid-configuration";
+    private static final String SLASH_WELL_KNOWN_OPENID_CONFIG = "/" + WELL_KNOWN_OPENID_CONFIG;
+
+    OpenIDProviderMetadataCache(ServiceConfiguration config, AsyncHttpClient httpClient, ApiClient apiClient) {
+        int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT);
+        int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS,
+                CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT);
+        int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS,
+                CACHE_EXPIRATION_SECONDS_DEFAULT);
+        this.httpClient = httpClient;
+        this.wellKnownApi = apiClient != null ? new WellKnownApi(apiClient) : null;
+        AsyncCacheLoader, OpenIDProviderMetadata> loader = (issuer, executor) -> {
+            if (issuer.isPresent()) {
+                return loadOpenIDProviderMetadataForIssuer(issuer.get());
+            } else {
+                return loadOpenIDProviderMetadataForKubernetesApiServer();
+            }
+        };
+        this.cache = Caffeine.newBuilder()
+                .maximumSize(maxSize)
+                .refreshAfterWrite(refreshAfterWriteSeconds, TimeUnit.SECONDS)
+                .expireAfterWrite(expireAfterSeconds, TimeUnit.SECONDS)
+                .buildAsync(loader);
+    }
+
+    /**
+     * Retrieve the OpenID Provider Metadata for the provided issuer.
+     * 

+     * Note: this method does not do any validation on the parameterized issuer. The OpenID Connect discovery
+     * spec requires that the issuer use the HTTPS scheme: https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata.
+     * The {@link AuthenticationProviderOpenID} class handles this verification.
+     *
+     * @param issuer - authority from which to retrieve the OpenID Provider Metadata
+     * @return the {@link OpenIDProviderMetadata} for the given issuer. Fail the completable future with
+     * AuthenticationException if any exceptions occur while retrieving the metadata.
+     */
+    CompletableFuture getOpenIDProviderMetadataForIssuer(@Nonnull String issuer) {
+        return cache.get(Optional.of(issuer));
+    }
+
+    /**
+     * A loader for the cache that retrieves the metadata from the issuer's /.well-known/openid-configuration endpoint.
+     * @return a connection to the issuer's /.well-known/openid-configuration endpoint. Fails with
+     * AuthenticationException if the URL is malformed or there is an exception while opening the connection
+     */
+    private CompletableFuture loadOpenIDProviderMetadataForIssuer(String issuer) {
+        String url;
+        // TODO URI's normalization likely follows RFC2396 (library doesn't say so explicitly), whereas the spec
+        //  https://openid.net/specs/openid-connect-discovery-1_0.html#NormalizationSteps
+        //  calls for normalization according to RFC3986, which is supposed to obsolete RFC2396. Is this a problem?
+        if (issuer.endsWith("/")) {
+            url = issuer + WELL_KNOWN_OPENID_CONFIG;
+        } else {
+            url = issuer + SLASH_WELL_KNOWN_OPENID_CONFIG;
+        }
+
+        return httpClient
+                .prepareGet(url)
+                .execute()
+                .toCompletableFuture()
+                .thenCompose(result -> {
+                    CompletableFuture future = new CompletableFuture<>();
+                    try {
+                        OpenIDProviderMetadata openIDProviderMetadata =
+                                reader.readValue(result.getResponseBodyAsBytes());
+                        // We can verify this issuer once and cache the result because the issuer uniquely maps
+                        // to the cached object.
+                        verifyIssuer(issuer, openIDProviderMetadata, false);
+                        future.complete(openIDProviderMetadata);
+                    } catch (AuthenticationException e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+                        future.completeExceptionally(e);
+                    } catch (Exception e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+                        future.completeExceptionally(new AuthenticationException(
+                                "Error retrieving OpenID Provider Metadata at " + issuer + ": " + e.getMessage()));
+                    }
+                    return future;
+                });
+    }
+
+    /**
+     * Retrieve the OpenID Provider Metadata for the Kubernetes API server. This method is used instead of
+     * {@link #getOpenIDProviderMetadataForIssuer(String)} because different validations are done. The Kubernetes
+     * API server does not technically implement the complete OIDC spec for discovery, but it does implement some of
+     * it, so this method validates what it can. Specifically, it skips validation that the Discovery Document
+     * provider's URI matches the issuer. It verifies that the issuer on the discovery document matches the issuer
+     * claim
+     * @return
+     */
+    CompletableFuture getOpenIDProviderMetadataForKubernetesApiServer(String issClaim) {
+        return cache.get(Optional.empty()).thenCompose(openIDProviderMetadata -> {
+            CompletableFuture future = new CompletableFuture<>();
+            try {
+                verifyIssuer(issClaim, openIDProviderMetadata, true);
+                future.complete(openIDProviderMetadata);
+            } catch (AuthenticationException e) {
+                incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+                future.completeExceptionally(e);
+            }
+            return future;
+        });
+    }
+
+    private CompletableFuture loadOpenIDProviderMetadataForKubernetesApiServer() {
+        CompletableFuture future = new CompletableFuture<>();
+        try {
+            wellKnownApi.getServiceAccountIssuerOpenIDConfigurationAsync(new ApiCallback<>() {
+                @Override
+                public void onFailure(ApiException e, int statusCode, Map> responseHeaders) {
+                    incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+                    future.completeExceptionally(new AuthenticationException(
+                            "Error retrieving OpenID Provider Metadata from Kubernetes API server: "
+                                    + e.getMessage()));
+                }
+
+                @Override
+                public void onSuccess(String result, int statusCode, Map> responseHeaders) {
+                    try {
+                        // Validation that the token's issuer matches the issuer returned by the api server must be done
+                        // after the cache load operation to ensure each token's issuer matches the fallback issuer
+                        OpenIDProviderMetadata openIDProviderMetadata = reader.readValue(result);
+                        future.complete(openIDProviderMetadata);
+                    } catch (Exception e) {
+                        incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+                        future.completeExceptionally(new AuthenticationException(
+                                "Error retrieving OpenID Provider Metadata from Kubernetes API Server: "
+                                        + e.getMessage()));
+                    }
+                }
+
+                @Override
+                public void onUploadProgress(long bytesWritten, long contentLength, boolean done) {
+
+                }
+
+                @Override
+                public void onDownloadProgress(long bytesRead, long contentLength, boolean done) {
+
+                }
+            });
+        } catch (ApiException e) {
+            incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+            future.completeExceptionally(new AuthenticationException(
+                    "Error retrieving OpenID Provider Metadata from Kubernetes API server: " + e.getMessage()));
+        }
+        return future;
+    }
+
+    /**
+     * Verify the issuer url, as required by the OpenID Connect spec:
+     *
+     * Per the OpenID Connect Discovery spec, the issuer value returned MUST be identical to the
+     * Issuer URL that was directly used to retrieve the configuration information. This MUST also
+     * be identical to the iss Claim value in ID Tokens issued from this Issuer.
+     * https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderConfigurationValidation
+     *
+     * @param issuer - the issuer used to retrieve the metadata
+     * @param metadata - the OpenID Provider Metadata
+     * @param isK8s - whether the issuer is represented by the Kubernetes API server. This affects error reporting.
+     * @throws AuthenticationException if the issuer does not exactly match the metadata issuer
+     */
+    private void verifyIssuer(@Nonnull String issuer, OpenIDProviderMetadata metadata,
+                              boolean isK8s) throws AuthenticationException {
+        if (!issuer.equals(metadata.getIssuer())) {
+            if (isK8s) {
+                incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER);
+                throw new AuthenticationException("Issuer not allowed: " + issuer);
+            } else {
+                incrementFailureMetric(AuthenticationExceptionCode.ISSUER_MISMATCH);
+                throw new AuthenticationException(String.format("Issuer URL mismatch: [%s] should match [%s]",
+                        issuer, metadata.getIssuer()));
+            }
+        }
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java
new file mode 100644
index 0000000000000..d28f255d1bea7
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
new file mode 100644
index 0000000000000..298492652c0a8
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import com.github.tomakehurst.wiremock.WireMockServer;
+import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.impl.DefaultJwtBuilder;
+import io.jsonwebtoken.security.Keys;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.interfaces.RSAPublicKey;
+import java.util.Base64;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import javax.naming.AuthenticationException;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.authentication.AuthenticationDataCommand;
+import org.apache.pulsar.broker.authentication.AuthenticationState;
+import org.apache.pulsar.common.api.AuthData;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * An integration test relying on WireMock to simulate an OpenID Connect provider.
+ */
+public class AuthenticationProviderOpenIDIntegrationTest {
+
+    AuthenticationProviderOpenID provider;
+    PrivateKey privateKey;
+
+    // These are the kid values for JWKs in the /keys endpoint
+    String validJwk = "valid";
+    String invalidJwk = "invalid";
+
+    // The valid issuer
+    String issuer;
+    String issuerWithTrailingSlash;
+    // This issuer is configured to return an issuer in the openid-configuration
+    // that does not match the issuer on the token
+    String issuerThatFails;
+    String issuerK8s;
+    WireMockServer server;
+
+    @BeforeClass
+    void beforeClass() throws IOException {
+
+        // Port matches the port supplied in the fakeKubeConfig.yaml resource, which makes the k8s integration
+        // tests work correctly.
+        server = new WireMockServer(wireMockConfig().port(0));
+        server.start();
+        issuer = server.baseUrl();
+        issuerWithTrailingSlash = issuer + "/trailing-slash/";
+        issuerThatFails = issuer + "/fail";
+        issuerK8s = issuer + "/k8s";
+
+        // Set up a correct openid-configuration
+        server.stubFor(
+                get(urlEqualTo("/.well-known/openid-configuration"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "%s",
+                                          "jwks_uri": "%s/keys"
+                                        }
+                                        """.replace("%s", server.baseUrl()))));
+
+        // Set up a correct openid-configuration that the k8s integration test can use
+        // NOTE: integration tests revealed that the k8s client adds a trailing slash to the openid-configuration
+        // endpoint.
+        // NOTE: the jwks_uri is ignored, so we supply one that would fail here to ensure that we are not implicitly
+        // relying on the jwks_uri.
+        server.stubFor(
+                get(urlEqualTo("/k8s/.well-known/openid-configuration/"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "%s",
+                                          "jwks_uri": "%s/no/keys/hosted/here"
+                                        }
+                                        """.formatted(issuer, issuer))));
+
+        // Set up a correct openid-configuration that has a trailing slash in the issuers URL. This is a
+        // behavior observed by Auth0. In this case, the token's iss claim also has a trailing slash.
+        // The server should normalize the URL and call the Authorization Server without the double slash.
+        // NOTE: the spec does not indicate that the jwks_uri must have the same prefix as the issuer, and that
+        // is used here to simplify the testing.
+        server.stubFor(
+                get(urlEqualTo("/trailing-slash/.well-known/openid-configuration"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "%s",
+                                          "jwks_uri": "%s/keys"
+                                        }
+                                        """.formatted(issuerWithTrailingSlash, issuer))));
+
+        // Set up an incorrect openid-configuration where issuer does not match
+        server.stubFor(
+                get(urlEqualTo("/fail/.well-known/openid-configuration"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "https://wrong-issuer.com",
+                                          "jwks_uri": "%s/keys"
+                                        }
+                                        """.formatted(server.baseUrl()))));
+
+        // Create the token key pair
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        privateKey = keyPair.getPrivate();
+        RSAPublicKey rsaPublicKey = (RSAPublicKey) keyPair.getPublic();
+        String n = Base64.getUrlEncoder().encodeToString(rsaPublicKey.getModulus().toByteArray());
+        String e = Base64.getUrlEncoder().encodeToString(rsaPublicKey.getPublicExponent().toByteArray());
+
+        // Set up JWKS endpoint with a valid and an invalid public key
+        // The url matches are for both the normal and the k8s endpoints
+        server.stubFor(
+                get(urlMatching( "/keys|/k8s/openid/v1/jwks/"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody(
+                                        """
+                                        {
+                                            "keys" : [
+                                                {
+                                                "kid":"%s",
+                                                "kty":"RSA",
+                                                "alg":"RS256",
+                                                "n":"%s",
+                                                "e":"%s"
+                                                },
+                                                {
+                                                "kid": "%s",
+                                                "kty":"RSA",
+                                                "n":"invalid-key",
+                                                "e":"AQAB"
+                                                }
+                                            ]
+                                        }
+                                        """.formatted(validJwk, n, e, invalidJwk))));
+
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer + "," + issuerWithTrailingSlash
+                + "," + issuerThatFails);
+
+        // Create the fake kube config file. This file is configured via the env vars and is written to the
+        // target directory so maven clean will remove it.
+        byte[] template = Files.readAllBytes(Path.of(System.getenv("KUBECONFIG_TEMPLATE")));
+        String kubeConfig = new String(template).replace("${WIRE_MOCK_PORT}", String.valueOf(server.port()));
+        Files.write(Path.of(System.getenv("KUBECONFIG")), kubeConfig.getBytes());
+
+        provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+    }
+
+    @AfterClass
+    void afterClass() {
+        server.stop();
+    }
+
+    @Test
+    public void testTokenWithValidJWK() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
+    }
+
+    @Test
+    public void testTokenWithTrailingSlashAndValidJWK() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer + "/trailing-slash/", role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
+    }
+
+    @Test
+    public void testTokenWithInvalidJWK() throws Exception {
+        String role = "superuser";
+        String token = generateToken(invalidJwk, issuer, role, "allowed-audience",0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testAuthorizationServerReturnsIncorrectIssuerInOpenidConnectConfiguration() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuerThatFails, role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testTokenWithInvalidAudience() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "invalid-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testTokenWithInvalidIssuer() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, "https://not-an-allowed-issuer.com", role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testKubernetesApiServerAsDiscoverTrustedIssuerSuccess() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER");
+        // Test requires that k8sIssuer is not in the allowed token issuers
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        // We use the normal issuer on the token because the /k8s endpoint is configured via the kube config file
+        // made as part of the test setup. The kube client then gets the issuer from the /k8s endpoint and discovers
+        // this issuer.
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
+
+        // Ensure that a subsequent token with a different issuer still fails due to invalid issuer exception
+        String token2 = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token2)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+            assertTrue(e.getCause().getMessage().contains("Issuer not allowed"),
+                    "Unexpected error message: " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testKubernetesApiServerAsDiscoverTrustedIssuerFailsDueToMismatchedIssuerClaim() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        String token = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+
+    @Test
+    public void testKubernetesApiServerAsDiscoverPublicKeySuccess() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS");
+        // Test requires that k8sIssuer is not in the allowed token issuers
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
+
+        // Ensure that a subsequent token with a different issuer still fails due to invalid issuer exception
+        String token2 = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token2)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+            assertTrue(e.getCause().getMessage().contains("Issuer not allowed"),
+                    "Unexpected error message: " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testKubernetesApiServerAsDiscoverPublicKeyFailsDueToMismatchedIssuerClaim() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        String token = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testAuthenticationStateOpenIDForValidToken() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L);
+        AuthenticationState state = provider.newAuthState(null, null, null);
+        AuthData result = state.authenticateAsync(AuthData.of(token.getBytes())).get();
+        assertNull(result);
+        assertEquals(state.getAuthRole(), role);
+        assertEquals(state.getAuthDataSource().getCommandData(), token);
+        assertFalse(state.isExpired());
+    }
+
+    @Test
+    public void testAuthenticationStateOpenIDForExpiredToken() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, -10000L);
+        AuthenticationState state = provider.newAuthState(null, null, null);
+        try {
+            state.authenticateAsync(AuthData.of(token.getBytes())).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testAuthenticationStateOpenIDForValidTokenWithNoExp() throws Exception {
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, null);
+        AuthenticationState state = provider.newAuthState(null, null, null);
+        try {
+            state.authenticateAsync(AuthData.of(token.getBytes())).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    @Test
+    public void testAuthenticationStateOpenIDForTokenExpiration() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer);
+        // Use the leeway to allow the token to pass validation and then fail expiration
+        props.setProperty(AuthenticationProviderOpenID.ACCEPTED_TIME_LEEWAY_SECONDS, "10");
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 0L);
+        AuthenticationState state = provider.newAuthState(null, null, null);
+        AuthData result = state.authenticateAsync(AuthData.of(token.getBytes())).get();
+        assertNull(result);
+        assertEquals(state.getAuthRole(), role);
+        assertEquals(state.getAuthDataSource().getCommandData(), token);
+        assertTrue(state.isExpired());
+    }
+
+    @Test
+    void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer);
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "test");
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build a JWT with a custom claim
+        HashMap claims = new HashMap();
+        claims.put("test", "my-role");
+        String token = generateToken(validJwk, issuer, "not-my-role", "allowed-audience", 0L,
+                0L, 10000L, claims);
+        assertEquals(provider.authenticateAsync(new AuthenticationDataCommand(token)).get(), "my-role");
+    }
+
+    @Test
+    void ensureRoleClaimForNonSubClaimFailsWhenClaimIsMissing() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer);
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "test");
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build a JWT without the "test" claim, which should cause the authentication to fail
+        String token = generateToken(validJwk, issuer, "not-my-role", "allowed-audience", 0L,
+                0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
+        }
+    }
+
+    // This test is somewhat counterintuitive. We allow the state object to change roles, but then we fail it
+    // in the ServerCnx handling of the state object. As such, it is essential that the state object allow
+    // the role to change.
+    @Test
+    public void testAuthenticationStateOpenIDAllowsRoleChange() throws Exception {
+        String role1 = "superuser";
+        String token1 = generateToken(validJwk, issuer, role1, "allowed-audience", 0L, 0L, 10000L);
+        String role2 = "otheruser";
+        String token2 = generateToken(validJwk, issuer, role2, "allowed-audience", 0L, 0L, 10000L);
+        AuthenticationState state = provider.newAuthState(null, null, null);
+        AuthData result1 = state.authenticateAsync(AuthData.of(token1.getBytes())).get();
+        assertNull(result1);
+        assertEquals(state.getAuthRole(), role1);
+        assertEquals(state.getAuthDataSource().getCommandData(), token1);
+        assertFalse(state.isExpired());
+
+        AuthData result2 = state.authenticateAsync(AuthData.of(token2.getBytes())).get();
+        assertNull(result2);
+        assertEquals(state.getAuthRole(), role2);
+        assertEquals(state.getAuthDataSource().getCommandData(), token2);
+        assertFalse(state.isExpired());
+    }
+
+    private String generateToken(String kid, String issuer, String subject, String audience,
+                                 Long iatOffset, Long nbfOffset, Long expOffset) {
+        return generateToken(kid, issuer, subject, audience, iatOffset, nbfOffset, expOffset, new HashMap<>());
+    }
+
+    private String generateToken(String kid, String issuer, String subject, String audience,
+                                 Long iatOffset, Long nbfOffset, Long expOffset, HashMap extraClaims) {
+        long now = System.currentTimeMillis();
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        defaultJwtBuilder.setHeaderParam("kid", kid);
+        defaultJwtBuilder.setHeaderParam("typ", "JWT");
+        defaultJwtBuilder.setHeaderParam("alg", "RS256");
+        defaultJwtBuilder.setIssuer(issuer);
+        defaultJwtBuilder.setSubject(subject);
+        defaultJwtBuilder.setAudience(audience);
+        defaultJwtBuilder.setIssuedAt(iatOffset != null ? new Date(now + iatOffset) : null);
+        defaultJwtBuilder.setNotBefore(nbfOffset != null ? new Date(now + nbfOffset) : null);
+        defaultJwtBuilder.setExpiration(expOffset != null ? new Date(now + expOffset) : null);
+        defaultJwtBuilder.addClaims(extraClaims);
+        defaultJwtBuilder.signWith(privateKey);
+        return defaultJwtBuilder.compact();
+    }
+
+}
diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java
new file mode 100644
index 0000000000000..74abffe9c38e8
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static org.testng.Assert.assertNull;
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.interfaces.DecodedJWT;
+import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.impl.DefaultJwtBuilder;
+import io.jsonwebtoken.security.Keys;
+import java.security.KeyPair;
+import java.sql.Date;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import javax.naming.AuthenticationException;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.authentication.AuthenticationDataCommand;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests to cover the AuthenticationProviderOpenID without any network calls.
+ * 

+ * This class only tests the verification of tokens. It does not test the integration to retrieve tokens
+ * from an identity provider. See {@link AuthenticationProviderOpenIDIntegrationTest} for the authorization
+ * server integration tests.
+ * 

+ * Note: this class uses the io.jsonwebtoken library here because it has more utilities than the auth0 library.
+ * The jsonwebtoken library makes it easy to generate key pairs for many algorithms, and it also has an enum
+ * that can be used to assert that unsupported algorithms properly fail validation.
+ */
+public class AuthenticationProviderOpenIDTest {
+
+    // https://www.rfc-editor.org/rfc/rfc7518#section-3.1
+    @DataProvider(name = "supportedAlgorithms")
+    public static Object[][] supportedAlgorithms() {
+        return new Object[][] {
+                { SignatureAlgorithm.RS256 },
+                { SignatureAlgorithm.RS384 },
+                { SignatureAlgorithm.RS512 },
+                { SignatureAlgorithm.ES256 },
+                { SignatureAlgorithm.ES384 },
+                { SignatureAlgorithm.ES512 }
+        };
+    }
+
+    // Provider to use in common tests that are not verifying the configuration of the provider itself.
+    AuthenticationProviderOpenID basicProvider;
+    final String basicProviderAudience = "my-special-audience";
+
+    @BeforeClass
+    public void setup() throws Exception {
+        Properties properties = new Properties();
+        properties.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience);
+        properties.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://my-issuer.com");
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setProperties(properties);
+        basicProvider = new AuthenticationProviderOpenID();
+        basicProvider.initialize(conf);
+    }
+
+    @Test
+    public void testNullToken() {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Assert.assertThrows(AuthenticationException.class,
+                () -> provider.authenticate(new AuthenticationDataCommand(null)));
+    }
+
+    @Test
+    public void testThatNullAlgFails() {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Assert.assertThrows(AuthenticationException.class,
+                () -> provider.verifyJWT(null, null, null));
+    }
+
+    @Test
+    public void testThatUnsupportedAlgsThrowExceptions() {
+        Set unsupportedAlgs = new HashSet<>(Set.of(SignatureAlgorithm.values()));
+        Arrays.stream(supportedAlgorithms()).map(o -> (SignatureAlgorithm) o[0]).toList()
+                .forEach(unsupportedAlgs::remove);
+        unsupportedAlgs.forEach(unsupportedAlg -> {
+            AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+            // We don't create a public key because it's irrelevant
+            Assert.assertThrows(AuthenticationException.class,
+                    () -> provider.verifyJWT(null, unsupportedAlg.getValue(), null));
+        });
+    }
+
+    @Test(dataProvider = "supportedAlgorithms")
+    public void testThatSupportedAlgsWork(SignatureAlgorithm alg) throws AuthenticationException {
+        KeyPair keyPair = Keys.keyPairFor(alg);
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience);
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+
+        // Convert to the right class
+        DecodedJWT expectedValue = JWT.decode(defaultJwtBuilder.compact());
+        DecodedJWT actualValue = basicProvider.verifyJWT(keyPair.getPublic(), alg.getValue(), expectedValue);
+        Assert.assertEquals(expectedValue, actualValue);
+    }
+
+    @Test
+    public void testThatSupportedAlgWithMismatchedPublicKeyFromDifferentAlgFamilyFails() {
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience);
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+        // Choose a different algorithm from a different alg family
+        Assert.assertThrows(AuthenticationException.class,
+                () -> provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.ES512.getValue(), jwt));
+    }
+
+    @Test
+    public void testThatSupportedAlgWithMismatchedPublicKeyFromSameAlgFamilyFails() {
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience);
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+        // Choose a different algorithm but within the same alg family as above
+        Assert.assertThrows(AuthenticationException.class,
+                () -> provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS512.getValue(), jwt));
+    }
+
+    @Test
+    public void ensureExpiredTokenFails() {
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience);
+        defaultJwtBuilder.setExpiration(Date.from(Instant.now().minusSeconds(60)));
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+        Assert.assertThrows(AuthenticationException.class,
+                () -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), jwt));
+    }
+
+    @Test
+    public void ensureFutureNBFFails() throws Exception {
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience);
+        // Override the exp set in the above method
+        defaultJwtBuilder.setNotBefore(Date.from(Instant.now().plusSeconds(60)));
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+        Assert.assertThrows(AuthenticationException.class,
+                () -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), jwt));
+    }
+
+    @Test
+    public void ensureFutureIATFails() throws Exception {
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience);
+        // Override the exp set in the above method
+        defaultJwtBuilder.setIssuedAt(Date.from(Instant.now().plusSeconds(60)));
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+        Assert.assertThrows(AuthenticationException.class,
+                () -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), jwt));
+    }
+
+    @Test
+    public void ensureRecentlyExpiredTokenWithinConfiguredLeewaySucceeds() throws Exception {
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+
+        // Set up the provider
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ACCEPTED_TIME_LEEWAY_SECONDS, "10");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "leewayAudience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://localhost:8080");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build the JWT with an only recently expired token
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        addValidMandatoryClaims(defaultJwtBuilder, "leewayAudience");
+        defaultJwtBuilder.setExpiration(Date.from(Instant.ofEpochMilli(System.currentTimeMillis() - 5000L)));
+        defaultJwtBuilder.signWith(keyPair.getPrivate());
+        DecodedJWT expectedValue = JWT.decode(defaultJwtBuilder.compact());
+
+        // Test the verification
+        DecodedJWT actualValue = null;
+        try {
+            actualValue = provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS256.getValue(), expectedValue);
+        } catch (Exception e) {
+            Assert.fail("Token verification should not have thrown an exception.", e);
+        }
+        Assert.assertEquals(expectedValue, actualValue);
+    }
+
+    @Test
+    public void ensureEmptyIssuersFailsInitialization() {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config));
+    }
+
+    @Test
+    public void ensureEmptyIssuersFailsInitializationWithDisabledDiscoveryMode() {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "DISABLED");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config));
+    }
+
+    @Test
+    public void ensureEmptyIssuersWithK8sTrustedIssuerEnabledPassesInitialization() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "my-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+    }
+
+    @Test
+    public void ensureEmptyIssuersWithK8sPublicKeyEnabledPassesInitialization() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "my-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "");
+        props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+    }
+
+    @Test
+    public void ensureNullIssuersFailsInitialization() {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        ServiceConfiguration config = new ServiceConfiguration();
+        // Make sure this still defaults to null.
+        assertNull(config.getProperties().get(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS));
+        Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config));
+    }
+
+    @Test
+    public void ensureInsecureIssuerFailsInitialization() {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com,http://myissuer.com");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config));
+    }
+
+    @Test void ensureMissingRoleClaimReturnsNull() throws Exception {
+        // Build an empty JWT
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        defaultJwtBuilder.setAudience(basicProviderAudience);
+        DecodedJWT jwtWithoutSub = JWT.decode(defaultJwtBuilder.compact());
+
+        // A JWT with an empty role claim must result in a null role
+        assertNull(basicProvider.getRole(jwtWithoutSub));
+    }
+
+    @Test void ensureRoleClaimForStringReturnsRole() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience);
+        props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "sub");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build an empty JWT
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        defaultJwtBuilder.setSubject("my-role");
+        defaultJwtBuilder.setAudience(basicProviderAudience);
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+
+        Assert.assertEquals("my-role", provider.getRole(jwt));
+    }
+
+    @Test void ensureRoleClaimForSingletonListReturnsRole() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience);
+        props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build an empty JWT
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        HashMap> claims = new HashMap();
+        claims.put("roles", Collections.singletonList("my-role"));
+        defaultJwtBuilder.setClaims(claims);
+        defaultJwtBuilder.setAudience(basicProviderAudience);
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+
+        Assert.assertEquals("my-role", provider.getRole(jwt));
+    }
+
+    @Test void ensureRoleClaimForMultiEntryListReturnsFirstRole() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, basicProviderAudience);
+        props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build an empty JWT
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        HashMap> claims = new HashMap<>();
+        claims.put("roles", Arrays.asList("my-role-1", "my-role-2"));
+        defaultJwtBuilder.setClaims(claims);
+        defaultJwtBuilder.setAudience(basicProviderAudience);
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+
+        Assert.assertEquals("my-role-1", provider.getRole(jwt));
+    }
+
+    @Test void ensureRoleClaimForEmptyListReturnsNull() throws Exception {
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        Properties props = new Properties();
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "no-role-audience-test");
+        props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        provider.initialize(config);
+
+        // Build an empty JWT
+        DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+        HashMap> claims = new HashMap<>();
+        claims.put("roles", Collections.emptyList());
+        defaultJwtBuilder.setClaims(claims);
+        defaultJwtBuilder.setAudience("no-role-audience-test");
+        DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact());
+
+        // A JWT with an empty list role claim must result in a null role
+        assertNull(provider.getRole(jwt));
+    }
+
+    // Method simplifies adding the required claims. For the tests that need to verify invalid values for these
+    // claims, it is sufficient to set the values after calling this method.
+    private void addValidMandatoryClaims(DefaultJwtBuilder defaultJwtBuilder, String audience) {
+        defaultJwtBuilder.setExpiration(Date.from(Instant.now().plusSeconds(60)));
+        defaultJwtBuilder.setNotBefore(Date.from(Instant.now()));
+        defaultJwtBuilder.setIssuedAt(Date.from(Instant.now()));
+        defaultJwtBuilder.setAudience(audience);
+        defaultJwtBuilder.setSubject("my-role");
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java
new file mode 100644
index 0000000000000..c0945ae43c3e8
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationStateOpenIDTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.AssertJUnit.fail;
+import javax.naming.AuthenticationException;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests primarily focused on failure scenarios for {@link AuthenticationStateOpenID}. The happy path is covered
+ * by {@link AuthenticationProviderOpenIDIntegrationTest}.
+ */
+public class AuthenticationStateOpenIDTest {
+    @Test
+    void getRoleOnAuthStateShouldFailIfNotAuthenticated() {
+        AuthenticationStateOpenID state = new AuthenticationStateOpenID(null, null, null);
+        assertFalse(state.isComplete());
+        assertThrows(AuthenticationException.class, state::getAuthRole);
+    }
+
+    @Test
+    void getAuthDataOnAuthStateShouldBeNullIfNotAuthenticated() {
+        AuthenticationStateOpenID state = new AuthenticationStateOpenID(null, null, null);
+        assertNull(state.getAuthDataSource());
+    }
+
+    // We override this behavior to make it clear that this provider is only meant to be used asynchronously.
+    @SuppressWarnings("deprecation")
+    @Test
+    void authenticateShouldThrowNotImplementedException() {
+        AuthenticationStateOpenID state = new AuthenticationStateOpenID(null, null, null);
+        try {
+            state.authenticate(null);
+            fail("Expected AuthenticationException to be thrown");
+        } catch (AuthenticationException e) {
+            assertEquals(e.getMessage(), "Not supported");
+        }
+    }
+}
diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java
new file mode 100644
index 0000000000000..ad06d4b0c109a
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/ConfigUtilsTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authentication.oidc;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.testng.annotations.Test;
+
+public class ConfigUtilsTest {
+
+    @Test
+    public void testGetConfigValueAsStringWorks() {
+        Properties props = new Properties();
+        props.setProperty("prop1", "audience");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        String actual = ConfigUtils.getConfigValueAsString(config, "prop1");
+        assertEquals("audience", actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsStringReturnsNullIfMissing() {
+        Properties props = new Properties();
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        String actual = ConfigUtils.getConfigValueAsString(config, "prop1");
+        assertNull(actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsStringWithDefaultWorks() {
+        Properties props = new Properties();
+        props.setProperty("prop1", "audience");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        String actual = ConfigUtils.getConfigValueAsString(config, "prop1", "default");
+        assertEquals("audience", actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsStringReturnsDefaultIfMissing() {
+        Properties props = new Properties();
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        String actual = ConfigUtils.getConfigValueAsString(config, "prop1", "default");
+        assertEquals("default", actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsSetReturnsWorks() {
+        Properties props = new Properties();
+        props.setProperty("prop1", "a, b,   c");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        Set actual = ConfigUtils.getConfigValueAsSet(config, "prop1");
+        // Trims all whitespace
+        assertEquals(Set.of("a", "b", "c"), actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsSetReturnsEmptySetIfMissing() {
+        Properties props = new Properties();
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        Set actual = ConfigUtils.getConfigValueAsSet(config, "prop1");
+        assertEquals(Collections.emptySet(), actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsSetReturnsEmptySetIfBlankString() {
+        Properties props = new Properties();
+        props.setProperty("prop1", " ");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        Set actual = ConfigUtils.getConfigValueAsSet(config, "prop1");
+        assertEquals(Collections.emptySet(), actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsIntegerWorks() {
+        Properties props = new Properties();
+        props.setProperty("prop1", "1234");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        int actual = ConfigUtils.getConfigValueAsInt(config, "prop1", 9);
+        assertEquals(1234, actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsIntegerReturnsDefaultIfNAN() {
+        Properties props = new Properties();
+        props.setProperty("prop1", "non-a-number");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        int actual = ConfigUtils.getConfigValueAsInt(config, "prop1", 9);
+        assertEquals(9, actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsIntegerReturnsDefaultIfMissingProp() {
+        Properties props = new Properties();
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        int actual = ConfigUtils.getConfigValueAsInt(config, "prop1", 10);
+        assertEquals(10, actual);
+    }
+
+    @Test
+    public void testGetConfigValueAsBooleanReturnsDefaultIfMissingProp() {
+        Properties props = new Properties();
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        boolean actualFalse = ConfigUtils.getConfigValueAsBoolean(config, "prop1", false);
+        assertFalse(actualFalse);
+        boolean actualTrue = ConfigUtils.getConfigValueAsBoolean(config, "prop1", true);
+        assertTrue(actualTrue);
+    }
+
+    @Test
+    public void testGetConfigValueAsBooleanWorks() {
+        Properties props = new Properties();
+        props.setProperty("prop1", "true");
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setProperties(props);
+        boolean actual = ConfigUtils.getConfigValueAsBoolean(config, "prop1", false);
+        assertTrue(actual);
+    }
+
+}
diff --git a/pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml b/pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml
new file mode 100644
index 0000000000000..ef3d373399d61
--- /dev/null
+++ b/pulsar-broker-auth-oidc/src/test/java/resources/fakeKubeConfig.yaml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+current-context: wire-mock-server
+apiVersion: v1
+clusters:
+- cluster:
+    api-version: v1
+    server: http://localhost:${WIRE_MOCK_PORT}/k8s
+  name: wire-mock-server
+contexts:
+- context:
+    cluster: wire-mock-server
+    namespace: pulsar
+    user: test-user
+  name: wire-mock-server
+kind: Config
+users:
+- name: test-user
+  user:
+    token: fake-token
\ No newline at end of file

From bafecb2a9c0e73942de6a38df72dd5888d5afd66 Mon Sep 17 00:00:00 2001
From: Matteo Merli 
Date: Mon, 10 Apr 2023 18:30:22 -0700
Subject: [PATCH 165/404] [improve] Removed usages of SafeRun (#20060)

### Motivation

With BK 4.16, we don't need to pass `SafeRunnable` instances to the `OrderedExecutor` anymore. The executor has embedded the logic of checking and logging exceptions.

Removing the SafeRun will avoid extra allocations in critical path and clutter of the code

### Verifying this change

- [ ] Make sure that the change passes the CI checks.

*(Please pick either of the following options)*

This change is a trivial rework / code cleanup without any test coverage.

*(or)*

This change is already covered by existing tests, such as *(please describe tests)*.

*(or)*

This change added tests and can be verified as follows:

*(example:)*
  - *Added integration tests for end-to-end deployment with large payloads (10MB)*
  - *Extended integration test for recovery after broker failure*

### Does this pull request potentially affect one of the following parts:



*If the box was checked, please highlight the changes*

- [ ] Dependencies (add or upgrade a dependency)
- [ ] The public API
- [ ] The schema
- [ ] The default values of configurations
- [ ] The threading model
- [ ] The binary protocol
- [ ] The REST endpoints
- [ ] The admin CLI options
- [ ] The metrics
- [ ] Anything that affects deployment

### Documentation



- [ ] `doc` 
- [ ] `doc-required` 
- [x] `doc-not-needed` 
- [ ] `doc-complete` 

### Matching PR in forked repository

PR in forked repository: (https://github.com/merlimat/pulsar/pull/6)


---
 .../mledger/impl/ManagedCursorImpl.java       | 21 +++--
 .../mledger/impl/ManagedLedgerImpl.java       | 44 +++++-----
 .../mledger/impl/MetaStoreImpl.java           | 27 +++---
 .../bookkeeper/mledger/impl/OpAddEntry.java   | 10 +--
 .../bookkeeper/mledger/impl/OpReadEntry.java  | 13 ++-
 .../mledger/impl/ShadowManagedLedgerImpl.java | 13 ++-
 .../cache/RangeEntryCacheManagerImpl.java     |  5 +-
 .../bookkeeper/mledger/util/SafeRun.java      | 57 ------------
 .../impl/ManagedCursorConcurrencyTest.java    |  5 +-
 .../prometheus/PrometheusMetricsServlet.java  |  5 +-
 .../pulsar/broker/service/BrokerService.java  | 87 +++++++++++--------
 .../pulsar/broker/service/ServerCnx.java      |  5 +-
 ...PersistentDispatcherMultipleConsumers.java | 11 ++-
 ...sistentDispatcherSingleActiveConsumer.java | 25 ++----
 ...tStickyKeyDispatcherMultipleConsumers.java |  5 +-
 ...tStreamingDispatcherMultipleConsumers.java | 12 ++-
 ...reamingDispatcherSingleActiveConsumer.java | 10 +--
 .../StreamingEntryReader.java                 | 19 ++--
 .../coordination/impl/LeaderElectionImpl.java |  3 +-
 19 files changed, 150 insertions(+), 227 deletions(-)
 delete mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 05cad09b01833..ef607fa7ed7cf 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -25,7 +25,6 @@
 import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.DEFAULT_LEDGER_DELETE_RETRIES;
 import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.createManagedLedgerException;
 import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException;
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.Collections2;
@@ -1358,7 +1357,7 @@ public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks
         final PositionImpl newPosition = (PositionImpl) newPos;
 
         // order trim and reset operations on a ledger
-        ledger.getExecutor().execute(safeRun(() -> {
+        ledger.getExecutor().execute(() -> {
             PositionImpl actualPosition = newPosition;
 
             if (!ledger.isValidPosition(actualPosition)
@@ -1375,7 +1374,7 @@ public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks
             }
 
             internalResetCursor(actualPosition, callback);
-        }));
+        });
     }
 
     @Override
@@ -2055,7 +2054,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) {
                         + "is later.", mdEntry.newPosition, persistentMarkDeletePosition);
             }
             // run with executor to prevent deadlock
-            ledger.getExecutor().execute(safeRun(() -> mdEntry.triggerComplete()));
+            ledger.getExecutor().execute(() -> mdEntry.triggerComplete());
             return;
         }
 
@@ -2074,7 +2073,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) {
                         + "in progress {} is later.", mdEntry.newPosition, inProgressLatest);
             }
             // run with executor to prevent deadlock
-            ledger.getExecutor().execute(safeRun(() -> mdEntry.triggerComplete()));
+            ledger.getExecutor().execute(() -> mdEntry.triggerComplete());
             return;
         }
 
@@ -2611,8 +2610,8 @@ private boolean shouldPersistUnackRangesToLedger() {
     private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map properties,
             MetaStoreCallback callback, boolean persistIndividualDeletedMessageRanges) {
         if (state == State.Closed) {
-            ledger.getExecutor().execute(safeRun(() -> callback.operationFailed(new MetaStoreException(
-                    new CursorAlreadyClosedException(name + " cursor already closed")))));
+            ledger.getExecutor().execute(() -> callback.operationFailed(new MetaStoreException(
+                    new CursorAlreadyClosedException(name + " cursor already closed"))));
             return;
         }
 
@@ -2845,7 +2844,7 @@ private CompletableFuture doCreateNewMetadataLedger() {
                 return;
             }
 
-            ledger.getExecutor().execute(safeRun(() -> {
+            ledger.getExecutor().execute(() -> {
                 ledger.mbean.endCursorLedgerCreateOp();
                 if (rc != BKException.Code.OK) {
                     log.warn("[{}] Error creating ledger for cursor {}: {}", ledger.getName(), name,
@@ -2858,7 +2857,7 @@ private CompletableFuture doCreateNewMetadataLedger() {
                     log.debug("[{}] Created ledger {} for cursor {}", ledger.getName(), lh.getId(), name);
                 }
                 future.complete(lh);
-            }));
+            });
         }, LedgerMetadataUtils.buildAdditionalMetadataForCursor(name));
 
         return future;
@@ -3192,7 +3191,7 @@ private void asyncDeleteLedger(final LedgerHandle lh, int retry) {
                 log.warn("[{}] Failed to delete ledger {}: {}", ledger.getName(), lh.getId(),
                         BKException.getMessage(rc));
                 if (!isNoSuchLedgerExistsException(rc)) {
-                    ledger.getScheduledExecutor().schedule(safeRun(() -> asyncDeleteLedger(lh, retry - 1)),
+                    ledger.getScheduledExecutor().schedule(() -> asyncDeleteLedger(lh, retry - 1),
                         DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS);
                 }
                 return;
@@ -3227,7 +3226,7 @@ private void asyncDeleteCursorLedger(int retry) {
                 log.warn("[{}][{}] Failed to delete ledger {}: {}", ledger.getName(), name, cursorLedger.getId(),
                         BKException.getMessage(rc));
                 if (!isNoSuchLedgerExistsException(rc)) {
-                    ledger.getScheduledExecutor().schedule(safeRun(() -> asyncDeleteCursorLedger(retry - 1)),
+                    ledger.getScheduledExecutor().schedule(() -> asyncDeleteCursorLedger(retry - 1),
                             DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS);
                 }
             }
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 8376ee1bb8467..c74db884a95a9 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -22,7 +22,6 @@
 import static com.google.common.base.Preconditions.checkState;
 import static java.lang.Math.min;
 import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException;
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.BoundType;
 import com.google.common.collect.Lists;
@@ -409,7 +408,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
                 if (!ledgers.isEmpty()) {
                     final long id = ledgers.lastKey();
                     OpenCallback opencb = (rc, lh, ctx1) -> {
-                        executor.execute(safeRun(() -> {
+                        executor.execute(() -> {
                             mbean.endDataLedgerOpenOp();
                             if (log.isDebugEnabled()) {
                                 log.debug("[{}] Opened ledger {}: {}", name, id, BKException.getMessage(rc));
@@ -439,7 +438,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
                                 callback.initializeFailed(createManagedLedgerException(rc));
                                 return;
                             }
-                        }));
+                        });
                     };
 
                     if (log.isDebugEnabled()) {
@@ -522,7 +521,7 @@ public void operationFailed(MetaStoreException e) {
                 return;
             }
 
-            executor.execute(safeRun(() -> {
+            executor.execute(() -> {
                 mbean.endDataLedgerCreateOp();
                 if (rc != BKException.Code.OK) {
                     callback.initializeFailed(createManagedLedgerException(rc));
@@ -551,7 +550,7 @@ public void operationFailed(MetaStoreException e) {
 
                 // Save it back to ensure all nodes exist
                 store.asyncUpdateLedgerIds(name, getManagedLedgerInfo(), ledgersStat, storeLedgersCb);
-            }));
+            });
         }, ledgerMetadata);
     }
 
@@ -774,10 +773,10 @@ public void asyncAddEntry(ByteBuf buffer, AddEntryCallback callback, Object ctx)
         buffer.retain();
 
         // Jump to specific thread to avoid contention from writers writing from different threads
-        executor.execute(safeRun(() -> {
+        executor.execute(() -> {
             OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, callback, ctx);
             internalAsyncAddEntry(addOperation);
-        }));
+        });
     }
 
     @Override
@@ -790,10 +789,10 @@ public void asyncAddEntry(ByteBuf buffer, int numberOfMessages, AddEntryCallback
         buffer.retain();
 
         // Jump to specific thread to avoid contention from writers writing from different threads
-        executor.execute(safeRun(() -> {
+        executor.execute(() -> {
             OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx);
             internalAsyncAddEntry(addOperation);
-        }));
+        });
     }
 
     protected synchronized void internalAsyncAddEntry(OpAddEntry addOperation) {
@@ -2374,7 +2373,7 @@ void notifyCursors() {
                 break;
             }
 
-            executor.execute(safeRun(waitingCursor::notifyEntriesAvailable));
+            executor.execute(waitingCursor::notifyEntriesAvailable);
         }
     }
 
@@ -2385,7 +2384,7 @@ void notifyWaitingEntryCallBacks() {
                 break;
             }
 
-            executor.execute(safeRun(cb::entriesAvailable));
+            executor.execute(cb::entriesAvailable);
         }
     }
 
@@ -2432,16 +2431,16 @@ private void trimConsumedLedgersInBackground() {
 
     @Override
     public void trimConsumedLedgersInBackground(CompletableFuture promise) {
-        executor.execute(safeRun(() -> internalTrimConsumedLedgers(promise)));
+        executor.execute(() -> internalTrimConsumedLedgers(promise));
     }
 
     public void trimConsumedLedgersInBackground(boolean isTruncate, CompletableFuture promise) {
-        executor.execute(safeRun(() -> internalTrimLedgers(isTruncate, promise)));
+        executor.execute(() -> internalTrimLedgers(isTruncate, promise));
     }
 
     private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture promise) {
-        scheduledExecutor.schedule(safeRun(() -> trimConsumedLedgersInBackground(isTruncate, promise)), 100,
-                TimeUnit.MILLISECONDS);
+        scheduledExecutor.schedule(() -> trimConsumedLedgersInBackground(isTruncate, promise),
+                100, TimeUnit.MILLISECONDS);
     }
 
     private void maybeOffloadInBackground(CompletableFuture promise) {
@@ -2456,7 +2455,7 @@ private void maybeOffloadInBackground(CompletableFuture promise) {
         final long offloadThresholdInSeconds =
                 Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInSeconds()).orElse(-1L);
         if (offloadThresholdInBytes >= 0 || offloadThresholdInSeconds >= 0) {
-            executor.execute(safeRun(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise)));
+            executor.execute(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise));
         }
     }
 
@@ -2477,7 +2476,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS
         }
 
         if (!offloadMutex.tryLock()) {
-            scheduledExecutor.schedule(safeRun(() -> maybeOffloadInBackground(finalPromise)),
+            scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise),
                     100, TimeUnit.MILLISECONDS);
             return;
         }
@@ -2956,7 +2955,7 @@ private void asyncDeleteLedger(long ledgerId, long retry) {
                 log.warn("[{}] Ledger was already deleted {}", name, ledgerId);
             } else if (rc != BKException.Code.OK) {
                 log.error("[{}] Error deleting ledger {} : {}", name, ledgerId, BKException.getMessage(rc));
-                scheduledExecutor.schedule(safeRun(() -> asyncDeleteLedger(ledgerId, retry - 1)),
+                scheduledExecutor.schedule(() -> asyncDeleteLedger(ledgerId, retry - 1),
                         DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS);
             } else {
                 if (log.isDebugEnabled()) {
@@ -3260,7 +3259,7 @@ private void tryTransformLedgerInfo(long ledgerId, LedgerInfoTransformation tran
             if (!metadataMutex.tryLock()) {
                 // retry in 100 milliseconds
                 scheduledExecutor.schedule(
-                        safeRun(() -> tryTransformLedgerInfo(ledgerId, transformation, finalPromise)), 100,
+                        () -> tryTransformLedgerInfo(ledgerId, transformation, finalPromise), 100,
                         TimeUnit.MILLISECONDS);
             } else { // lock acquired
                 CompletableFuture unlockingPromise = new CompletableFuture<>();
@@ -4011,9 +4010,8 @@ private void scheduleTimeoutTask() {
             timeoutSec = timeoutSec <= 0
                     ? Math.max(config.getAddEntryTimeoutSeconds(), config.getReadEntryTimeoutSeconds())
                     : timeoutSec;
-            this.timeoutTask = this.scheduledExecutor.scheduleAtFixedRate(safeRun(() -> {
-                checkTimeouts();
-            }), timeoutSec, timeoutSec, TimeUnit.SECONDS);
+            this.timeoutTask = this.scheduledExecutor.scheduleAtFixedRate(
+                    this::checkTimeouts, timeoutSec, timeoutSec, TimeUnit.SECONDS);
         }
     }
 
@@ -4336,7 +4334,7 @@ protected void updateLastLedgerCreatedTimeAndScheduleRolloverTask() {
                 checkLedgerRollTask.cancel(true);
             }
             this.checkLedgerRollTask = this.scheduledExecutor.schedule(
-                    safeRun(this::rollCurrentLedgerIfFull), this.maximumRolloverTimeMs, TimeUnit.MILLISECONDS);
+                    this::rollCurrentLedgerIfFull, this.maximumRolloverTimeMs, TimeUnit.MILLISECONDS);
         }
     }
 
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index fb93e929f50c7..fcce1f7766cc8 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -42,7 +42,6 @@
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo;
-import org.apache.bookkeeper.util.SafeRunnable;
 import org.apache.commons.lang.StringUtils;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.compression.CompressionCodec;
@@ -156,7 +155,7 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map
                 .exceptionally(ex -> {
                     try {
                         executor.executeOrdered(ledgerName,
-                                SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex))));
+                                () -> callback.operationFailed(getException(ex)));
                     } catch (RejectedExecutionException e) {
                         //executor maybe shutdown, use common pool to run callback.
                         CompletableFuture.runAsync(() -> callback.operationFailed(getException(ex)));
@@ -204,8 +203,8 @@ public void asyncUpdateLedgerIds(String ledgerName, ManagedLedgerInfo mlInfo, St
                 .thenAcceptAsync(newVersion -> callback.operationComplete(null, newVersion),
                         executor.chooseThread(ledgerName))
                 .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                    executor.executeOrdered(ledgerName,
+                            () -> callback.operationFailed(getException(ex)));
                     return null;
                 });
     }
@@ -221,8 +220,8 @@ public void getCursors(String ledgerName, MetaStoreCallback> callba
                 .thenAcceptAsync(cursors -> callback.operationComplete(cursors, null), executor
                         .chooseThread(ledgerName))
                 .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                    executor.executeOrdered(ledgerName,
+                            () -> callback.operationFailed(getException(ex)));
                     return null;
                 });
     }
@@ -249,8 +248,8 @@ public void asyncGetCursorInfo(String ledgerName, String cursorName,
                     }
                 }, executor.chooseThread(ledgerName))
                 .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                    executor.executeOrdered(ledgerName,
+                            () -> callback.operationFailed(getException(ex)));
                     return null;
                 });
     }
@@ -284,8 +283,8 @@ public void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedC
                 .thenAcceptAsync(optStat -> callback.operationComplete(null, optStat), executor
                         .chooseThread(ledgerName))
                 .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                    executor.executeOrdered(ledgerName,
+                            () -> callback.operationFailed(getException(ex)));
                     return null;
                 });
     }
@@ -303,7 +302,7 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal
                     callback.operationComplete(null, null);
                 }, executor.chooseThread(ledgerName))
                 .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> {
+                    executor.executeOrdered(ledgerName, () -> {
                         Throwable actEx = FutureUtil.unwrapCompletionException(ex);
                         if (actEx instanceof MetadataStoreException.NotFoundException){
                             log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName);
@@ -311,7 +310,7 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal
                             return;
                         }
                         callback.operationFailed(getException(ex));
-                    }));
+                    });
                     return null;
                 });
     }
@@ -329,8 +328,8 @@ public void removeManagedLedger(String ledgerName, MetaStoreCallback callb
                     callback.operationComplete(null, null);
                 }, executor.chooseThread(ledgerName))
                 .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                    executor.executeOrdered(ledgerName,
+                            () -> callback.operationFailed(getException(ex)));
                     return null;
                 });
     }
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java
index c56123c24cac1..ae2beafb64374 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java
@@ -35,8 +35,6 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor;
-import org.apache.bookkeeper.mledger.util.SafeRun;
-import org.apache.bookkeeper.util.SafeRunnable;
 
 
 /**
@@ -44,7 +42,7 @@
  *
  */
 @Slf4j
-public class OpAddEntry extends SafeRunnable implements AddCallback, CloseCallback {
+public class OpAddEntry implements AddCallback, CloseCallback, Runnable {
     protected ManagedLedgerImpl ml;
     LedgerHandle ledger;
     private long entryId;
@@ -212,7 +210,7 @@ public void addComplete(int rc, final LedgerHandle lh, long entryId, Object ctx)
 
     // Called in executor hashed on managed ledger name, once the add operation is complete
     @Override
-    public void safeRun() {
+    public void run() {
         if (payloadProcessorHandle != null) {
             payloadProcessorHandle.release();
         }
@@ -328,11 +326,11 @@ void handleAddFailure(final LedgerHandle lh) {
         ManagedLedgerImpl finalMl = this.ml;
         finalMl.mbean.recordAddEntryError();
 
-        finalMl.getExecutor().execute(SafeRun.safeRun(() -> {
+        finalMl.getExecutor().execute(() -> {
             // Force the creation of a new ledger. Doing it in a background thread to avoid acquiring ML lock
             // from a BK callback.
             finalMl.ledgerClosed(lh);
-        }));
+        });
     }
 
     void close() {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
index 81b14359514b9..19211553a5f74 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
@@ -18,7 +18,6 @@
  */
 package org.apache.bookkeeper.mledger.impl;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import io.netty.util.Recycler;
 import io.netty.util.Recycler.Handle;
 import java.util.ArrayList;
@@ -108,10 +107,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
 
         if (!entries.isEmpty()) {
             // There were already some entries that were read before, we can return them
-            cursor.ledger.getExecutor().execute(safeRun(() -> {
+            cursor.ledger.getExecutor().execute(() -> {
                 callback.readEntriesComplete(entries, ctx);
                 recycle();
-            }));
+            });
         } else if (cursor.config.isAutoSkipNonRecoverableData() && exception instanceof NonRecoverableLedgerException) {
             log.warn("[{}][{}] read failed from ledger at position:{} : {}", cursor.ledger.getName(), cursor.getName(),
                     readPosition, exception.getMessage());
@@ -161,20 +160,20 @@ void checkReadCompletion() {
                 && maxPosition.compareTo(readPosition) > 0) {
 
             // We still have more entries to read from the next ledger, schedule a new async operation
-            cursor.ledger.getExecutor().execute(safeRun(() -> {
+            cursor.ledger.getExecutor().execute(() -> {
                 readPosition = cursor.ledger.startReadOperationOnLedger(nextReadPosition);
                 cursor.ledger.asyncReadEntries(OpReadEntry.this);
-            }));
+            });
         } else {
             // The reading was already completed, release resources and trigger callback
             try {
                 cursor.readOperationCompleted();
 
             } finally {
-                cursor.ledger.getExecutor().execute(safeRun(() -> {
+                cursor.ledger.getExecutor().execute(() -> {
                     callback.readEntriesComplete(entries, ctx);
                     recycle();
-                }));
+                });
             }
         }
     }
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java
index b1f239413472f..b33dd87543f77 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java
@@ -19,7 +19,6 @@
 package org.apache.bookkeeper.mledger.impl;
 
 import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException;
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -66,13 +65,13 @@ public ShadowManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper book
     @Override
     synchronized void initialize(ManagedLedgerInitializeLedgerCallback callback, Object ctx) {
         log.info("Opening shadow managed ledger {} with source={}", name, sourceMLName);
-        executor.execute(safeRun(() -> doInitialize(callback, ctx)));
+        executor.execute(() -> doInitialize(callback, ctx));
     }
 
     private void doInitialize(ManagedLedgerInitializeLedgerCallback callback, Object ctx) {
         // Fetch the list of existing ledgers in the source managed ledger
         store.watchManagedLedgerInfo(sourceMLName, (managedLedgerInfo, stat) ->
-                executor.execute(safeRun(() -> processSourceManagedLedgerInfo(managedLedgerInfo, stat)))
+                executor.execute(() -> processSourceManagedLedgerInfo(managedLedgerInfo, stat))
         );
         store.getManagedLedgerInfo(sourceMLName, false, null, new MetaStore.MetaStoreCallback<>() {
             @Override
@@ -106,7 +105,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat)
 
                 final long lastLedgerId = ledgers.lastKey();
                 mbean.startDataLedgerOpenOp();
-                AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> executor.execute(safeRun(() -> {
+                AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> executor.execute(() -> {
                     mbean.endDataLedgerOpenOp();
                     if (log.isDebugEnabled()) {
                         log.debug("[{}] Opened source ledger {}", name, lastLedgerId);
@@ -145,7 +144,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat)
                                 BKException.getMessage(rc));
                         callback.initializeFailed(createManagedLedgerException(rc));
                     }
-                }));
+                });
                 //open ledger in readonly mode.
                 bookKeeper.asyncOpenLedgerNoRecovery(lastLedgerId, digestType, config.getPassword(), opencb, null);
 
@@ -317,7 +316,7 @@ private synchronized void processSourceManagedLedgerInfo(MLDataFormats.ManagedLe
             mbean.startDataLedgerOpenOp();
             //open ledger in readonly mode.
             bookKeeper.asyncOpenLedgerNoRecovery(lastLedgerId, digestType, config.getPassword(),
-                    (rc, lh, ctx1) -> executor.execute(safeRun(() -> {
+                    (rc, lh, ctx1) -> executor.execute(() -> {
                         mbean.endDataLedgerOpenOp();
                         if (log.isDebugEnabled()) {
                             log.debug("[{}] Opened new source ledger {}", name, lastLedgerId);
@@ -342,7 +341,7 @@ private synchronized void processSourceManagedLedgerInfo(MLDataFormats.ManagedLe
                             log.error("[{}] Failed to open source ledger {}: {}", name, lastLedgerId,
                                     BKException.getMessage(rc));
                         }
-                    })), null);
+                    }), null);
         }
 
         //handle old ledgers deleted.
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
index 080c70b5873cd..d5a3019855cb5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
@@ -18,7 +18,6 @@
  */
 package org.apache.bookkeeper.mledger.impl.cache;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import com.google.common.collect.Lists;
 import io.netty.buffer.ByteBuf;
 import java.util.concurrent.ConcurrentHashMap;
@@ -116,7 +115,7 @@ boolean hasSpaceInCache() {
 
         // Trigger a single eviction in background. While the eviction is running we stop inserting entries in the cache
         if (currentSize > evictionTriggerThreshold && evictionInProgress.compareAndSet(false, true)) {
-            mlFactory.getScheduledExecutor().execute(safeRun(() -> {
+            mlFactory.getScheduledExecutor().execute(() -> {
                 // Trigger a new cache eviction cycle to bring the used memory below the cacheEvictionWatermark
                 // percentage limit
                 long sizeToEvict = currentSize - (long) (maxSize * cacheEvictionWatermark);
@@ -136,7 +135,7 @@ boolean hasSpaceInCache() {
                     mlFactoryMBean.recordCacheEviction();
                     evictionInProgress.set(false);
                 }
-            }));
+            });
         }
 
         return currentSize < maxSize;
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java
deleted file mode 100644
index 570cb7ae735ab..0000000000000
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/SafeRun.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.util;
-
-import java.util.function.Consumer;
-import org.apache.bookkeeper.util.SafeRunnable;
-
-/**
- * Static builders for {@link SafeRunnable}s.
- */
-public class SafeRun {
-    public static SafeRunnable safeRun(Runnable runnable) {
-        return new SafeRunnable() {
-            @Override
-            public void safeRun() {
-                runnable.run();
-            }
-        };
-    }
-
-    /**
-     *
-     * @param runnable
-     * @param exceptionHandler
-     *            handler that will be called when there are any exception
-     * @return
-     */
-    public static SafeRunnable safeRun(Runnable runnable, Consumer exceptionHandler) {
-        return new SafeRunnable() {
-            @Override
-            public void safeRun() {
-                try {
-                    runnable.run();
-                } catch (Throwable t) {
-                    exceptionHandler.accept(t);
-                    throw t;
-                }
-            }
-        };
-    }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
index 3fa0234e13a55..7558f07db76ca 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
@@ -18,7 +18,6 @@
  */
 package org.apache.bookkeeper.mledger.impl;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNull;
@@ -383,7 +382,7 @@ public void testConcurrentIndividualDeletesWithGetNthEntry() throws Exception {
         final AtomicInteger iteration = new AtomicInteger(0);
 
         for (int i = 0; i < deleteEntries; i++) {
-            executor.submit(safeRun(() -> {
+            executor.submit(() -> {
                 try {
                     cursor.asyncDelete(addedEntries.get(iteration.getAndIncrement()), new DeleteCallback() {
                         @Override
@@ -403,7 +402,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
                 } finally {
                     counter.countDown();
                 }
-            }));
+            });
         }
 
         counter.await();
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
index 136fe77d77aff..64d1fcdab6f14 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.stats.prometheus;
 
-import static org.apache.bookkeeper.util.SafeRunnable.safeRun;
 import io.netty.util.concurrent.DefaultThreadFactory;
 import java.io.EOFException;
 import java.io.IOException;
@@ -61,7 +60,7 @@ public void init() throws ServletException {
     protected void doGet(HttpServletRequest request, HttpServletResponse response) {
         AsyncContext context = request.startAsync();
         context.setTimeout(metricsServletTimeoutMs);
-        executor.execute(safeRun(() -> {
+        executor.execute(() -> {
             long start = System.currentTimeMillis();
             HttpServletResponse res = (HttpServletResponse) context.getResponse();
             try {
@@ -92,7 +91,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) {
                             + "this is likely due to metricsServletTimeoutMs: {} ms elapsed: {}", time, e + "");
                 }
             }
-        }));
+        });
     }
 
     protected void generateMetrics(String cluster, ServletOutputStream outputStream) throws IOException {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index a6345f4a56a71..bb08734298f04 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -20,7 +20,6 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static org.apache.bookkeeper.mledger.ManagedLedgerConfig.PROPERTY_SOURCE_TOPIC_KEY;
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import static org.apache.commons.collections4.CollectionUtils.isEmpty;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName;
@@ -56,7 +55,6 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
@@ -76,6 +74,7 @@
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.bookkeeper.common.util.OrderedExecutor;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback;
 import org.apache.bookkeeper.mledger.LedgerOffloader;
@@ -324,8 +323,10 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws
         this.acceptorGroup = EventLoopUtil.newEventLoopGroup(
                 pulsar.getConfiguration().getNumAcceptorThreads(), false, acceptorThreadFactory);
         this.workerGroup = eventLoopGroup;
-        this.statsUpdater = Executors.newSingleThreadScheduledExecutor(
-                new ExecutorProvider.ExtendedThreadFactory("pulsar-stats-updater"));
+        this.statsUpdater = OrderedScheduler.newSchedulerBuilder()
+                .name("pulsar-stats-updater")
+                .numThreads(1)
+                .build();
         this.authorizationService = new AuthorizationService(
                 pulsar.getConfiguration(), pulsar().getPulsarResources());
         this.entryFilterProvider = new EntryFilterProvider(pulsar.getConfiguration());
@@ -333,22 +334,32 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws
         pulsar.getLocalMetadataStore().registerListener(this::handleMetadataChanges);
         pulsar.getConfigurationMetadataStore().registerListener(this::handleMetadataChanges);
 
-        this.inactivityMonitor = Executors.newSingleThreadScheduledExecutor(
-                new ExecutorProvider.ExtendedThreadFactory("pulsar-inactivity-monitor"));
-        this.messageExpiryMonitor = Executors.newSingleThreadScheduledExecutor(
-                new ExecutorProvider.ExtendedThreadFactory("pulsar-msg-expiry-monitor"));
-        this.compactionMonitor =
-                Executors.newSingleThreadScheduledExecutor(
-                        new ExecutorProvider.ExtendedThreadFactory("pulsar-compaction-monitor"));
-        this.consumedLedgersMonitor = Executors.newSingleThreadScheduledExecutor(
-                new ExecutorProvider.ExtendedThreadFactory("consumed-Ledgers-monitor"));
+
+        this.inactivityMonitor = OrderedScheduler.newSchedulerBuilder()
+                .name("pulsar-inactivity-monitor")
+                .numThreads(1)
+                .build();
+        this.messageExpiryMonitor = OrderedScheduler.newSchedulerBuilder()
+                .name("pulsar-msg-expiry-monitor")
+                .numThreads(1)
+                .build();
+        this.compactionMonitor = OrderedScheduler.newSchedulerBuilder()
+                .name("pulsar-compaction-monitor")
+                .numThreads(1)
+                .build();
+        this.consumedLedgersMonitor = OrderedScheduler.newSchedulerBuilder()
+                .name("pulsar-consumed-ledgers-monitor")
+                .numThreads(1)
+                .build();
         this.topicPublishRateLimiterMonitor =
                 new PublishRateLimiterMonitor("pulsar-topic-publish-rate-limiter-monitor");
         this.brokerPublishRateLimiterMonitor =
                 new PublishRateLimiterMonitor("pulsar-broker-publish-rate-limiter-monitor");
         this.backlogQuotaManager = new BacklogQuotaManager(pulsar);
-        this.backlogQuotaChecker = Executors.newSingleThreadScheduledExecutor(
-                new ExecutorProvider.ExtendedThreadFactory("pulsar-backlog-quota-checker"));
+        this.backlogQuotaChecker = OrderedScheduler.newSchedulerBuilder()
+                .name("pulsar-backlog-quota-checker")
+                .numThreads(1)
+                .build();
         this.authenticationService = new AuthenticationService(pulsar.getConfiguration());
         this.blockedDispatchers =
                 ConcurrentOpenHashSet.newBuilder().build();
@@ -367,7 +378,7 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws
             log.info("Enabling per-broker unack-message limit {} and dispatcher-limit {} on blocked-broker",
                     maxUnackedMessages, maxUnackedMsgsPerDispatcher);
             // block misbehaving dispatcher by checking periodically
-            pulsar.getExecutor().scheduleAtFixedRate(safeRun(this::checkUnAckMessageDispatching),
+            pulsar.getExecutor().scheduleAtFixedRate(this::checkUnAckMessageDispatching,
                     600, 30, TimeUnit.SECONDS);
         } else {
             this.maxUnackedMessages = 0;
@@ -557,7 +568,7 @@ public void start() throws Exception {
     }
 
     protected void startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpdateFrequencyInSecs) {
-        statsUpdater.scheduleAtFixedRate(safeRun(this::updateRates),
+        statsUpdater.scheduleAtFixedRate(this::updateRates,
             statsUpdateInitialDelayInSecs, statsUpdateFrequencyInSecs, TimeUnit.SECONDS);
 
         // Ensure the broker starts up with initial stats
@@ -567,11 +578,12 @@ protected void startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpd
     protected void startDeduplicationSnapshotMonitor() {
         int interval = pulsar().getConfiguration().getBrokerDeduplicationSnapshotFrequencyInSeconds();
         if (interval > 0 && pulsar().getConfiguration().isBrokerDeduplicationEnabled()) {
-            this.deduplicationSnapshotMonitor =
-                    Executors.newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory(
-                            "deduplication-snapshot-monitor"));
-            deduplicationSnapshotMonitor.scheduleAtFixedRate(safeRun(() -> forEachTopic(
-                    Topic::checkDeduplicationSnapshot))
+            this.deduplicationSnapshotMonitor = OrderedScheduler.newSchedulerBuilder()
+                    .name("deduplication-snapshot-monitor")
+                    .numThreads(1)
+                    .build();
+            deduplicationSnapshotMonitor.scheduleAtFixedRate(() -> forEachTopic(
+                    Topic::checkDeduplicationSnapshot)
                     , interval, interval, TimeUnit.SECONDS);
         }
     }
@@ -579,14 +591,14 @@ protected void startDeduplicationSnapshotMonitor() {
     protected void startInactivityMonitor() {
         if (pulsar().getConfiguration().isBrokerDeleteInactiveTopicsEnabled()) {
             int interval = pulsar().getConfiguration().getBrokerDeleteInactiveTopicsFrequencySeconds();
-            inactivityMonitor.scheduleAtFixedRate(safeRun(() -> checkGC()), interval, interval,
+            inactivityMonitor.scheduleAtFixedRate(() -> checkGC(), interval, interval,
                     TimeUnit.SECONDS);
         }
 
         // Deduplication info checker
         long duplicationCheckerIntervalInSeconds = TimeUnit.MINUTES
                 .toSeconds(pulsar().getConfiguration().getBrokerDeduplicationProducerInactivityTimeoutMinutes()) / 3;
-        inactivityMonitor.scheduleAtFixedRate(safeRun(this::checkMessageDeduplicationInfo),
+        inactivityMonitor.scheduleAtFixedRate(this::checkMessageDeduplicationInfo,
                 duplicationCheckerIntervalInSeconds,
                 duplicationCheckerIntervalInSeconds, TimeUnit.SECONDS);
 
@@ -595,7 +607,7 @@ protected void startInactivityMonitor() {
             long subscriptionExpiryCheckIntervalInSeconds =
                     TimeUnit.MINUTES.toSeconds(pulsar().getConfiguration()
                             .getSubscriptionExpiryCheckIntervalInMinutes());
-            inactivityMonitor.scheduleAtFixedRate(safeRun(this::checkInactiveSubscriptions),
+            inactivityMonitor.scheduleAtFixedRate(this::checkInactiveSubscriptions,
                     subscriptionExpiryCheckIntervalInSeconds,
                     subscriptionExpiryCheckIntervalInSeconds, TimeUnit.SECONDS);
         }
@@ -603,21 +615,21 @@ protected void startInactivityMonitor() {
         // check cluster migration
         int interval = pulsar().getConfiguration().getClusterMigrationCheckDurationSeconds();
         if (interval > 0) {
-            inactivityMonitor.scheduleAtFixedRate(safeRun(() -> checkClusterMigration()), interval, interval,
+            inactivityMonitor.scheduleAtFixedRate(() -> checkClusterMigration(), interval, interval,
                     TimeUnit.SECONDS);
         }
     }
 
     protected void startMessageExpiryMonitor() {
         int interval = pulsar().getConfiguration().getMessageExpiryCheckIntervalInMinutes();
-        messageExpiryMonitor.scheduleAtFixedRate(safeRun(this::checkMessageExpiry), interval, interval,
+        messageExpiryMonitor.scheduleAtFixedRate(this::checkMessageExpiry, interval, interval,
                 TimeUnit.MINUTES);
     }
 
     protected void startCheckReplicationPolicies() {
         int interval = pulsar.getConfig().getReplicationPolicyCheckDurationSeconds();
         if (interval > 0) {
-            messageExpiryMonitor.scheduleAtFixedRate(safeRun(this::checkReplicationPolicies), interval, interval,
+            messageExpiryMonitor.scheduleAtFixedRate(this::checkReplicationPolicies, interval, interval,
                     TimeUnit.SECONDS);
         }
     }
@@ -625,16 +637,16 @@ protected void startCheckReplicationPolicies() {
     protected void startCompactionMonitor() {
         int interval = pulsar().getConfiguration().getBrokerServiceCompactionMonitorIntervalInSeconds();
         if (interval > 0) {
-            compactionMonitor.scheduleAtFixedRate(safeRun(() -> checkCompaction()),
-                                                  interval, interval, TimeUnit.SECONDS);
+            compactionMonitor.scheduleAtFixedRate(this::checkCompaction,
+                    interval, interval, TimeUnit.SECONDS);
         }
     }
 
     protected void startConsumedLedgersMonitor() {
         int interval = pulsar().getConfiguration().getRetentionCheckIntervalInSeconds();
         if (interval > 0) {
-            consumedLedgersMonitor.scheduleAtFixedRate(safeRun(this::checkConsumedLedgers),
-                                                            interval, interval, TimeUnit.SECONDS);
+            consumedLedgersMonitor.scheduleAtFixedRate(this::checkConsumedLedgers,
+                    interval, interval, TimeUnit.SECONDS);
         }
     }
 
@@ -642,7 +654,7 @@ protected void startBacklogQuotaChecker() {
         if (pulsar().getConfiguration().isBacklogQuotaCheckEnabled()) {
             final int interval = pulsar().getConfiguration().getBacklogQuotaCheckIntervalInSeconds();
             log.info("Scheduling a thread to check backlog quota after [{}] seconds in background", interval);
-            backlogQuotaChecker.scheduleAtFixedRate(safeRun(this::monitorBacklogQuota), interval, interval,
+            backlogQuotaChecker.scheduleAtFixedRate(this::monitorBacklogQuota, interval, interval,
                     TimeUnit.SECONDS);
         } else {
             log.info("Backlog quota check monitoring is disabled");
@@ -702,12 +714,15 @@ synchronized void startOrUpdate(long tickTimeMs, Runnable checkTask, Runnable re
                 stop();
             }
             //start monitor.
-            scheduler = Executors.newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory(name));
+            scheduler = OrderedScheduler.newSchedulerBuilder()
+                    .name(name)
+                    .numThreads(1)
+                    .build();
             // schedule task that sums up publish-rate across all cnx on a topic ,
             // and check the rate limit exceeded or not.
-            scheduler.scheduleAtFixedRate(safeRun(checkTask), tickTimeMs, tickTimeMs, TimeUnit.MILLISECONDS);
+            scheduler.scheduleAtFixedRate(checkTask, tickTimeMs, tickTimeMs, TimeUnit.MILLISECONDS);
             // schedule task that refreshes rate-limiting bucket
-            scheduler.scheduleAtFixedRate(safeRun(refreshTask), 1, 1, TimeUnit.SECONDS);
+            scheduler.scheduleAtFixedRate(refreshTask, 1, 1, TimeUnit.SECONDS);
             this.tickTimeMs = tickTimeMs;
             this.refreshTask = refreshTask;
         }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 677ed25ef2163..bfd7c001e4cfb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -67,7 +67,6 @@
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.bookkeeper.mledger.util.SafeRun;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.lang3.mutable.MutableInt;
@@ -1674,9 +1673,9 @@ protected void handleSend(CommandSend send, ByteBuf headersAndPayload) {
                 final long producerId = send.getProducerId();
                 final long sequenceId = send.getSequenceId();
                 final long highestSequenceId = send.getHighestSequenceId();
-                service.getTopicOrderedExecutor().executeOrdered(producer.getTopic().getName(), SafeRun.safeRun(() -> {
+                service.getTopicOrderedExecutor().executeOrdered(producer.getTopic().getName(), () -> {
                     commandSender.sendSendReceiptResponse(producerId, sequenceId, highestSequenceId, -1, -1);
-                }));
+                });
                 producer.recordMessageDrop(send.getNumMessages());
                 return;
             } else {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 9540f6efe0005..4ac755860fc7b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Range;
@@ -236,9 +235,9 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE
 
     @Override
     public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) {
-        topic.getBrokerService().executor().execute(safeRun(() -> {
+        topic.getBrokerService().executor().execute(() -> {
             internalConsumerFlow(consumer, additionalNumberOfMessages);
-        }));
+        });
     }
 
     private synchronized void internalConsumerFlow(Consumer consumer, int additionalNumberOfMessages) {
@@ -264,7 +263,7 @@ private synchronized void internalConsumerFlow(Consumer consumer, int additional
      *
      */
     public void readMoreEntriesAsync() {
-        topic.getBrokerService().executor().execute(safeRun(this::readMoreEntries));
+        topic.getBrokerService().executor().execute(this::readMoreEntries);
     }
 
     public synchronized void readMoreEntries() {
@@ -584,14 +583,14 @@ public final synchronized void readEntriesComplete(List entries, Object c
             // setting sendInProgress here, because sendMessagesToConsumers will be executed
             // in a separate thread, and we want to prevent more reads
             acquireSendInProgress();
-            dispatchMessagesThread.execute(safeRun(() -> {
+            dispatchMessagesThread.execute(() -> {
                 if (sendMessagesToConsumers(readType, entries, false)) {
                     updatePendingBytesToDispatch(-size);
                     readMoreEntries();
                 } else {
                     updatePendingBytesToDispatch(-size);
                 }
-            }));
+            });
         } else {
             if (sendMessagesToConsumers(readType, entries, true)) {
                 updatePendingBytesToDispatch(-size);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
index 03dce58af3a92..7cbf7bd2c787a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
@@ -38,7 +38,6 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException;
 import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.bookkeeper.mledger.util.SafeRun;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer;
 import org.apache.pulsar.broker.service.Consumer;
@@ -149,9 +148,7 @@ protected void cancelPendingRead() {
 
     @Override
     public void readEntriesComplete(final List entries, Object obj) {
-        topicExecutor.execute(SafeRun.safeRun(() -> {
-            internalReadEntriesComplete(entries, obj);
-        }));
+        topicExecutor.execute(() -> internalReadEntriesComplete(entries, obj));
     }
 
     public synchronized void internalReadEntriesComplete(final List entries, Object obj) {
@@ -229,21 +226,19 @@ protected void dispatchEntriesToConsumer(Consumer currentConsumer, List e
                             sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes());
 
                     // Schedule a new read batch operation only after the previous batch has been written to the socket.
-                    topicExecutor.execute(SafeRun.safeRun(() -> {
+                    topicExecutor.execute(() -> {
                             synchronized (PersistentDispatcherSingleActiveConsumer.this) {
                                 Consumer newConsumer = getActiveConsumer();
                                 readMoreEntries(newConsumer);
                             }
-                        }));
+                        });
                 }
             });
     }
 
     @Override
     public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) {
-        topicExecutor.execute(SafeRun.safeRun(() -> {
-            internalConsumerFlow(consumer);
-        }));
+        topicExecutor.execute(() -> internalConsumerFlow(consumer));
     }
 
     private synchronized void internalConsumerFlow(Consumer consumer) {
@@ -272,9 +267,7 @@ private synchronized void internalConsumerFlow(Consumer consumer) {
 
     @Override
     public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) {
-        topicExecutor.execute(SafeRun.safeRun(() -> {
-            internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch);
-        }));
+        topicExecutor.execute(() -> internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch));
     }
 
     private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) {
@@ -466,9 +459,7 @@ protected Pair calculateToRead(Consumer consumer) {
 
     @Override
     public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
-        topicExecutor.execute(SafeRun.safeRun(() -> {
-            internalReadEntriesFailed(exception, ctx);
-        }));
+        topicExecutor.execute(() -> internalReadEntriesFailed(exception, ctx));
     }
 
     private synchronized void internalReadEntriesFailed(ManagedLedgerException exception, Object ctx) {
@@ -516,7 +507,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep
         topic.getBrokerService().executor().schedule(() -> {
 
             // Jump again into dispatcher dedicated thread
-            topicExecutor.execute(SafeRun.safeRun(() -> {
+            topicExecutor.execute(() -> {
                 synchronized (PersistentDispatcherSingleActiveConsumer.this) {
                     Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
                     // we should retry the read if we have an active consumer and there is no pending read
@@ -533,7 +524,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep
                                 currentConsumer, havePendingRead);
                     }
                 }
-            }));
+            });
         }, waitTimeMillis, TimeUnit.MILLISECONDS);
 
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index a0292729e734f..1a8c6e180a2a2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import com.google.common.annotations.VisibleForTesting;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.util.ArrayList;
@@ -415,7 +414,7 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en
     public void markDeletePositionMoveForward() {
         // Execute the notification in different thread to avoid a mutex chain here
         // from the delete operation that was completed
-        topic.getBrokerService().getTopicOrderedExecutor().execute(safeRun(() -> {
+        topic.getBrokerService().getTopicOrderedExecutor().execute(() -> {
             synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) {
                 if (recentlyJoinedConsumers != null && !recentlyJoinedConsumers.isEmpty()
                         && removeConsumersFromRecentJoinedConsumers()) {
@@ -424,7 +423,7 @@ && removeConsumersFromRecentJoinedConsumers()) {
                     readMoreEntries();
                 }
             }
-        }));
+        });
     }
 
     private boolean removeConsumersFromRecentJoinedConsumers() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java
index 5df1fc2c6db26..44e8a423344cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import com.google.common.collect.Lists;
 import java.util.Set;
 import java.util.concurrent.Executor;
@@ -30,7 +29,6 @@
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.bookkeeper.mledger.util.SafeRun;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.streamingdispatch.PendingReadEntryRequest;
@@ -105,13 +103,13 @@ public synchronized void readEntryComplete(Entry entry, PendingReadEntryRequest
             // setting sendInProgress here, because sendMessagesToConsumers will be executed
             // in a separate thread, and we want to prevent more reads
             acquireSendInProgress();
-            dispatchMessagesThread.execute(safeRun(() -> {
+            dispatchMessagesThread.execute(() -> {
                 if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), false)) {
                     readMoreEntries();
                 } else {
                     updatePendingBytesToDispatch(-size);
                 }
-            }));
+            });
         } else {
             if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), true)) {
                 readMoreEntriesAsync();
@@ -129,7 +127,7 @@ public synchronized void readEntryComplete(Entry entry, PendingReadEntryRequest
     public void canReadMoreEntries(boolean withBackoff) {
         havePendingRead = false;
         topic.getBrokerService().executor().schedule(() -> {
-            topicExecutor.execute(SafeRun.safeRun(() -> {
+            topicExecutor.execute(() -> {
                 synchronized (PersistentStreamingDispatcherMultipleConsumers.this) {
                     if (!havePendingRead) {
                         log.info("[{}] Scheduling read operation", name);
@@ -138,7 +136,7 @@ public void canReadMoreEntries(boolean withBackoff) {
                         log.info("[{}] Skipping read since we have pendingRead", name);
                     }
                 }
-            }));
+            });
         }, withBackoff
                 ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS);
     }
@@ -216,7 +214,7 @@ public synchronized void readMoreEntries() {
                     havePendingReplayRead = false;
                     // We should not call readMoreEntries() recursively in the same thread
                     // as there is a risk of StackOverflowError
-                    topic.getBrokerService().executor().execute(safeRun(this::readMoreEntries));
+                    topic.getBrokerService().executor().execute(this::readMoreEntries);
                 }
             } else if (BLOCKED_DISPATCHER_ON_UNACKMSG_UPDATER.get(this) == TRUE) {
                 log.debug("[{}] Dispatcher read is blocked due to unackMessages {} reached to max {}", name,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java
index 9000850ed69ed..efe9de778a3e7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
 import com.google.common.collect.Lists;
 import java.util.concurrent.Executor;
@@ -29,7 +28,6 @@
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.bookkeeper.mledger.util.SafeRun;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.EntryBatchIndexesAcks;
@@ -68,7 +66,7 @@ public PersistentStreamingDispatcherSingleActiveConsumer(ManagedCursor cursor, S
     public void canReadMoreEntries(boolean withBackoff) {
         havePendingRead = false;
         topic.getBrokerService().executor().schedule(() -> {
-             topicExecutor.execute(SafeRun.safeRun(() -> {
+             topicExecutor.execute(() -> {
                 synchronized (PersistentStreamingDispatcherSingleActiveConsumer.this) {
                     Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
                     if (currentConsumer != null && !havePendingRead) {
@@ -81,7 +79,7 @@ public void canReadMoreEntries(boolean withBackoff) {
                                 currentConsumer, havePendingRead);
                     }
                 }
-            }));
+            });
         }, withBackoff
                 ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS);
     }
@@ -115,9 +113,7 @@ public String getName() {
      */
     @Override
     public void readEntryComplete(Entry entry, PendingReadEntryRequest ctx) {
-        dispatcherExecutor.execute(safeRun(() -> {
-            internalReadEntryComplete(entry, ctx);
-        }));
+        dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx));
     }
 
     public synchronized void internalReadEntryComplete(Entry entry, PendingReadEntryRequest ctx) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java
index 0f75538ee88f0..6ffc5ba0f623a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java
@@ -35,7 +35,6 @@
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.bookkeeper.mledger.util.SafeRun;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException;
 import org.apache.pulsar.client.impl.Backoff;
@@ -157,9 +156,7 @@ public synchronized void asyncReadEntries(int numEntriesToRead, long maxReadSize
     @Override
     public void readEntryComplete(Entry entry, Object ctx) {
         // Don't block caller thread, complete read entry with dispatcher dedicated thread.
-        dispatcherExecutor.execute(SafeRun.safeRun(() -> {
-            internalReadEntryComplete(entry, ctx);
-        }));
+        dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx));
     }
 
     private void internalReadEntryComplete(Entry entry, Object ctx) {
@@ -198,9 +195,7 @@ private void internalReadEntryComplete(Entry entry, Object ctx) {
     @Override
     public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
         // Don't block caller thread, complete read entry fail with dispatcher dedicated thread.
-        dispatcherExecutor.execute(SafeRun.safeRun(() -> {
-            internalReadEntryFailed(exception, ctx);
-        }));
+        dispatcherExecutor.execute(() -> internalReadEntryFailed(exception, ctx));
     }
 
     private void internalReadEntryFailed(ManagedLedgerException exception, Object ctx) {
@@ -257,13 +252,13 @@ private void internalCancelReadRequests() {
     public boolean cancelReadRequests() {
         if (STATE_UPDATER.compareAndSet(this, State.Issued, State.Canceling)) {
             // Don't block caller thread, complete cancel read with dispatcher dedicated thread.
-             topicExecutor.execute(SafeRun.safeRun(() -> {
+             topicExecutor.execute(() -> {
                 synchronized (StreamingEntryReader.this) {
                     if (STATE_UPDATER.compareAndSet(this, State.Canceling, State.Canceled)) {
                         internalCancelReadRequests();
                     }
                 }
-            }));
+            });
             return true;
         }
         return false;
@@ -282,16 +277,16 @@ private void cleanQueue(Queue queue) {
     private void retryReadRequest(PendingReadEntryRequest pendingReadEntryRequest, long delay) {
         topic.getBrokerService().executor().schedule(() -> {
             // Jump again into dispatcher dedicated thread
-            dispatcherExecutor.execute(SafeRun.safeRun(() -> {
+            dispatcherExecutor.execute(() -> {
                 ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger();
                 managedLedger.asyncReadEntry(pendingReadEntryRequest.position, this, pendingReadEntryRequest);
-            }));
+            });
         }, delay, TimeUnit.MILLISECONDS);
     }
 
     @Override
     public void entriesAvailable() {
-        dispatcherExecutor.execute(SafeRun.safeRun(this::internalEntriesAvailable));
+        dispatcherExecutor.execute(this::internalEntriesAvailable);
     }
 
     private synchronized void internalEntriesAvailable() {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index 11ae62226e7cd..9e6a9b94c42a3 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -30,7 +30,6 @@
 import java.util.function.Consumer;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.common.concurrent.FutureUtils;
-import org.apache.bookkeeper.common.util.SafeRunnable;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.GetResult;
 import org.apache.pulsar.metadata.api.MetadataCache;
@@ -89,7 +88,7 @@ private enum InternalState {
         this.sequencer = FutureUtil.Sequencer.create();
         store.registerListener(this::handlePathNotification);
         store.registerSessionListener(this::handleSessionNotification);
-        updateCachedValueFuture = executor.scheduleWithFixedDelay(SafeRunnable.safeRun(this::getLeaderValue),
+        updateCachedValueFuture = executor.scheduleWithFixedDelay(this::getLeaderValue,
                 metadataCacheConfig.getRefreshAfterWriteMillis() / 2,
                 metadataCacheConfig.getRefreshAfterWriteMillis(), TimeUnit.MILLISECONDS);
     }

From a91865140893f7e9737f6ce9ffc052584c721884 Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Tue, 11 Apr 2023 10:25:57 +0800
Subject: [PATCH 166/404] [improve][client] PIP-224: Add getLastMessageIds API
 (#20040)

Co-authored-by: Baodi Shi 
---
 .../pulsar/client/api/TopicReaderTest.java    |  6 +++
 .../client/impl/TopicsConsumerImplTest.java   | 37 +++++++++++++++++++
 .../apache/pulsar/client/api/Consumer.java    | 21 +++++++++++
 .../pulsar/client/impl/ConsumerBase.java      | 15 ++++++++
 .../pulsar/client/impl/ConsumerImpl.java      |  7 ++++
 .../client/impl/MultiMessageIdImpl.java       |  1 +
 .../client/impl/MultiTopicsConsumerImpl.java  | 13 +++++++
 7 files changed, 100 insertions(+)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java
index c2eb957ee605d..424081b904c81 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java
@@ -1096,6 +1096,12 @@ public void testHasMessageAvailable() throws Exception {
         assertFalse(lastMsgId instanceof BatchMessageIdImpl);
         assertEquals(lastMsgId.getLedgerId(), messageId.getLedgerId());
         assertEquals(lastMsgId.getEntryId(), messageId.getEntryId());
+        List lastMsgIds = reader.getConsumer().getLastMessageIds();
+        assertEquals(lastMsgIds.size(), 1);
+        assertEquals(lastMsgIds.get(0).getOwnerTopic(), topicName);
+        MessageIdAdv lastMsgIdAdv = (MessageIdAdv) lastMsgIds.get(0);
+        assertEquals(lastMsgIdAdv.getLedgerId(), messageId.getLedgerId());
+        assertEquals(lastMsgIdAdv.getEntryId(), messageId.getEntryId());
         reader.close();
 
         CountDownLatch latch = new CountDownLatch(numOfMessage);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
index ce4a0ae86ac4e..73fe97996424c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
@@ -33,6 +33,7 @@
 import org.apache.pulsar.client.api.ConsumerEventListener;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.MessageRouter;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
@@ -42,7 +43,9 @@
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.client.api.TopicMetadata;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
 import org.apache.pulsar.common.policies.data.SubscriptionStats;
@@ -1097,6 +1100,11 @@ public void testGetLastMessageId() throws Exception {
         admin.topics().createPartitionedTopic(topicName2, 2);
         admin.topics().createPartitionedTopic(topicName3, 3);
 
+        final Set topics = new HashSet<>();
+        topics.add(topicName1);
+        IntStream.range(0, 2).forEach(i -> topics.add(topicName2 + TopicName.PARTITIONED_TOPIC_SUFFIX + i));
+        IntStream.range(0, 3).forEach(i -> topics.add(topicName3 + TopicName.PARTITIONED_TOPIC_SUFFIX + i));
+
         // 1. producer connect
         Producer producer1 = pulsarClient.newProducer().topic(topicName1)
             .enableBatching(false)
@@ -1146,12 +1154,27 @@ public void testGetLastMessageId() throws Exception {
             }
         });
 
+        List msgIds = consumer.getLastMessageIds();
+        assertEquals(msgIds.size(), 6);
+        assertEquals(msgIds.stream().map(TopicMessageId::getOwnerTopic).collect(Collectors.toSet()), topics);
+        for (TopicMessageId msgId : msgIds) {
+            int numMessages = (int) ((MessageIdAdv) msgId).getEntryId() + 1;
+            if (msgId.getOwnerTopic().equals(topicName1)) {
+                assertEquals(numMessages, totalMessages);
+            } else if (msgId.getOwnerTopic().startsWith(topicName2)) {
+                assertEquals(numMessages, totalMessages / 2);
+            } else {
+                assertEquals(numMessages, totalMessages / 3);
+            }
+        }
+
         for (int i = 0; i < totalMessages; i++) {
             producer1.send((messagePredicate + "producer1-" + i).getBytes());
             producer2.send((messagePredicate + "producer2-" + i).getBytes());
             producer3.send((messagePredicate + "producer3-" + i).getBytes());
         }
 
+
         messageId = consumer.getLastMessageId();
         assertTrue(messageId instanceof MultiMessageIdImpl);
         MultiMessageIdImpl multiMessageId2 = (MultiMessageIdImpl) messageId;
@@ -1170,6 +1193,20 @@ public void testGetLastMessageId() throws Exception {
             }
         });
 
+        msgIds = consumer.getLastMessageIds();
+        assertEquals(msgIds.size(), 6);
+        assertEquals(msgIds.stream().map(TopicMessageId::getOwnerTopic).collect(Collectors.toSet()), topics);
+        for (TopicMessageId msgId : msgIds) {
+            int numMessages = (int) ((MessageIdAdv) msgId).getEntryId() + 1;
+            if (msgId.getOwnerTopic().equals(topicName1)) {
+                assertEquals(numMessages, totalMessages * 2);
+            } else if (msgId.getOwnerTopic().startsWith(topicName2)) {
+                assertEquals(numMessages, totalMessages);
+            } else {
+                assertEquals(numMessages, totalMessages / 3 * 2);
+            }
+        }
+
         consumer.unsubscribe();
         consumer.close();
         producer1.close();
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java
index 694099004965a..88ad24fe1f484 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.client.api;
 
 import java.io.Closeable;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
@@ -539,16 +540,36 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message,
      * Get the last message id available for consume.
      *
      * @return the last message id.
+     * @apiNote If the consumer is a multi-topics consumer, the returned value cannot be used anywhere.
+     * @deprecated Use {@link Consumer#getLastMessageIds()} instead.
      */
+    @Deprecated
     MessageId getLastMessageId() throws PulsarClientException;
 
     /**
      * Get the last message id available for consume.
      *
      * @return a future that can be used to track the completion of the operation.
+     * @deprecated Use {@link Consumer#getLastMessageIdsAsync()}} instead.
      */
+    @Deprecated
     CompletableFuture getLastMessageIdAsync();
 
+    /**
+     * Get all the last message id of the topics the consumer subscribed.
+     *
+     * @return the list of TopicMessageId instances of all the topics that the consumer subscribed
+     * @throws PulsarClientException if failed to get last message id.
+     * @apiNote It's guaranteed that the owner topic of each TopicMessageId in the returned list is different from owner
+     *   topics of other TopicMessageId instances
+     */
+    List getLastMessageIds() throws PulsarClientException;
+
+    /**
+     * The asynchronous version of {@link Consumer#getLastMessageIds()}.
+     */
+    CompletableFuture> getLastMessageIdsAsync();
+
     /**
      * @return Whether the consumer is connected to the broker
      */
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
index 973b3302f4199..0db2a8e0ab9f5 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
@@ -53,6 +53,7 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.client.api.transaction.Transaction;
 import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
 import org.apache.pulsar.client.impl.transaction.TransactionImpl;
@@ -730,6 +731,7 @@ public void close() throws PulsarClientException {
     public abstract CompletableFuture closeAsync();
 
 
+    @Deprecated
     @Override
     public MessageId getLastMessageId() throws PulsarClientException {
         try {
@@ -742,9 +744,22 @@ public MessageId getLastMessageId() throws PulsarClientException {
         }
     }
 
+    @Deprecated
     @Override
     public abstract CompletableFuture getLastMessageIdAsync();
 
+    @Override
+    public List getLastMessageIds() throws PulsarClientException {
+        try {
+            return getLastMessageIdsAsync().get();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw PulsarClientException.unwrap(e);
+        } catch (ExecutionException e) {
+            throw PulsarClientException.unwrap(e);
+        }
+    }
+
     private boolean isCumulativeAcknowledgementAllowed(SubscriptionType type) {
         return SubscriptionType.Shared != type && SubscriptionType.Key_Shared != type;
     }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index fb372566426d3..cc01609319698 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -2336,11 +2336,18 @@ private static final class GetLastMessageIdResponse {
         }
     }
 
+    @Deprecated
     @Override
     public CompletableFuture getLastMessageIdAsync() {
         return internalGetLastMessageIdAsync().thenApply(r -> r.lastMessageId);
     }
 
+    @Override
+    public CompletableFuture> getLastMessageIdsAsync() {
+        return getLastMessageIdAsync()
+                .thenApply(msgId -> Collections.singletonList(TopicMessageId.create(topic, msgId)));
+    }
+
     public CompletableFuture internalGetLastMessageIdAsync() {
         if (getState() == State.Closing || getState() == State.Closed) {
             return FutureUtil
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java
index 6e60239ffe537..f40e3476dd0e0 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiMessageIdImpl.java
@@ -29,6 +29,7 @@
  * This is useful when MessageId is need for partition/multi-topics/pattern consumer.
  * e.g. seek(), ackCumulative(), getLastMessageId().
  */
+@Deprecated
 public class MultiMessageIdImpl implements MessageId {
     @Getter
     private Map map;
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index 5fe0e4a82b840..ef0345de919ea 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -1468,6 +1468,7 @@ public Timeout getPartitionsAutoUpdateTimeout() {
         return partitionsAutoUpdateTimeout;
     }
 
+    @Deprecated
     @Override
     public CompletableFuture getLastMessageIdAsync() {
         CompletableFuture returnFuture = new CompletableFuture<>();
@@ -1496,6 +1497,18 @@ public CompletableFuture getLastMessageIdAsync() {
         return returnFuture;
     }
 
+    @Override
+    public CompletableFuture> getLastMessageIdsAsync() {
+        final List>> futures = consumers.values().stream()
+                .map(ConsumerImpl::getLastMessageIdsAsync)
+                .collect(Collectors.toList());
+        return FutureUtil.waitForAll(futures).thenApply(__ -> {
+            final List messageIds = new ArrayList<>();
+            futures.stream().map(CompletableFuture::join).forEach(messageIds::addAll);
+            return messageIds;
+        });
+    }
+
     private static final Logger log = LoggerFactory.getLogger(MultiTopicsConsumerImpl.class);
 
     public static boolean isIllegalMultiTopicsMessageId(MessageId messageId) {

From 25dea3dc9dd6f5c2956cb7b9376450515f225a6b Mon Sep 17 00:00:00 2001
From: Mathieu Grenonville 
Date: Tue, 11 Apr 2023 04:52:11 +0200
Subject: [PATCH 167/404] [fix] Use scheduled executor in
 BinaryProtoLookupService (#20043)

Motivation: PulsarClient using ExecutorService as ScheduledExecutorService

Modifications: Use exactly ScheduledExecutorService
---
 .../java/org/apache/pulsar/client/impl/PulsarClientImpl.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
index f37709f3d84f6..6c749a8cf4354 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
@@ -205,7 +205,7 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG
                 lookup = new HttpLookupService(conf, this.eventLoopGroup);
             } else {
                 lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(),
-                        conf.isUseTls(), this.externalExecutorProvider.getExecutor());
+                        conf.isUseTls(), this.scheduledExecutorProvider.getExecutor());
             }
             if (timer == null) {
                 this.timer = new HashedWheelTimer(getThreadFactory("pulsar-timer"), 1, TimeUnit.MILLISECONDS);

From b7f7e040b6ad27827ba5aa31bf51be621d506560 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 11 Apr 2023 14:18:51 +0800
Subject: [PATCH 168/404] [fix] [admin] PIP-259: Admin API can not work if uri
 too large (#19514)

Motivation: If automatic topic creation is enabled, producers and consumers can create topics with long names, but once created, such topics cannot be managed by the Admin API, it will respond "414 URI is too large".

Modifications: add a config `pulsar.conf.httpMaxRequestHeaderSize` to make the jetty can accept the request with a longer Uri
---
 conf/broker.conf                              |  5 ++
 conf/proxy.conf                               |  5 ++
 .../pulsar/broker/ServiceConfiguration.java   | 11 ++++
 .../apache/pulsar/broker/web/WebService.java  |  8 ++-
 .../naming/ServiceConfigurationTest.java      |  1 +
 .../configurations/pulsar_broker_test.conf    |  1 +
 .../proxy/server/AdminProxyHandler.java       |  4 +-
 .../proxy/server/ProxyConfiguration.java      | 12 ++++
 .../apache/pulsar/proxy/server/WebServer.java | 19 +++++-
 .../server/AuthedAdminProxyHandlerTest.java   | 28 +++++++++
 .../proxy/server/ProxyConfigurationTest.java  |  2 +
 .../proxy/server/ProxyIsAHttpProxyTest.java   | 59 ++++++++++++++++++-
 12 files changed, 150 insertions(+), 5 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 414756abcb7cb..292b1ef1683b7 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -903,6 +903,11 @@ saslJaasServerRoleTokenSignerSecretPath=
 # If >0, it will reject all HTTP requests with bodies larged than the configured limit
 httpMaxRequestSize=-1
 
+# The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger
+# form content encoded in a URL.However, larger headers consume more memory and can make a server more vulnerable to
+# denial of service attacks.
+httpMaxRequestHeaderSize = 8192
+
 # If true, the broker will reject all HTTP requests using the TRACE and TRACK verbs.
 # This setting may be necessary if the broker is deployed into an environment that uses http port
 # scanning and flags web servers allowing the TRACE method as insecure.
diff --git a/conf/proxy.conf b/conf/proxy.conf
index a5110ae57471a..a33db8bbf7a30 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -277,6 +277,11 @@ maxHttpServerConnections=2048
 # Max concurrent web requests
 maxConcurrentHttpRequests=1024
 
+# The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger
+# form content encoded in a URL.However, larger headers consume more memory and can make a server more vulnerable to
+# denial of service attacks.
+httpMaxRequestHeaderSize = 8192
+
 ## Configure the datasource of basic authenticate, supports the file and Base64 format.
 # file:
 # basicAuthConf=/path/my/.htpasswd
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index cea159b74fd1d..b31c86560f848 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1584,6 +1584,17 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     )
     private long httpMaxRequestSize = -1;
 
+    @FieldContext(
+            category = CATEGORY_HTTP,
+            doc = """
+                The maximum size in bytes of the request header.
+                Larger headers will allow for more and/or larger cookies plus larger form content encoded in a URL.
+                However, larger headers consume more memory and can make a server more vulnerable to denial of service
+                attacks.
+              """
+    )
+    private int httpMaxRequestHeaderSize = 8 * 1024;
+
     @FieldContext(
         category =  CATEGORY_HTTP,
         doc = "If true, the broker will reject all HTTP requests using the TRACE and TRACK verbs.\n"
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
index 96e7c9d556b71..2d6a6af58477e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
@@ -33,6 +33,8 @@
 import org.apache.pulsar.jetty.tls.JettySslContextFactory;
 import org.eclipse.jetty.server.ConnectionLimit;
 import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.ContextHandler;
@@ -100,8 +102,10 @@ public WebService(PulsarService pulsar) throws PulsarServerException {
         List connectors = new ArrayList<>();
 
         Optional port = config.getWebServicePort();
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        httpConfig.setRequestHeaderSize(pulsar.getConfig().getHttpMaxRequestHeaderSize());
         if (port.isPresent()) {
-            httpConnector = new ServerConnector(server);
+            httpConnector = new ServerConnector(server, new HttpConnectionFactory(httpConfig));
             httpConnector.setPort(port.get());
             httpConnector.setHost(pulsar.getBindAddress());
             connectors.add(httpConnector);
@@ -140,7 +144,7 @@ public WebService(PulsarService pulsar) throws PulsarServerException {
                             config.getWebServiceTlsProtocols(),
                             config.getTlsCertRefreshCheckDurationSec());
                 }
-                httpsConnector = new ServerConnector(server, sslCtxFactory);
+                httpsConnector = new ServerConnector(server, sslCtxFactory, new HttpConnectionFactory(httpConfig));
                 httpsConnector.setPort(tlsPort.get());
                 httpsConnector.setHost(pulsar.getBindAddress());
                 connectors.add(httpsConnector);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java
index 2142900194731..55971c15adf68 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java
@@ -72,6 +72,7 @@ public void testInit() throws Exception {
         assertEquals(config.getMaxMessagePublishBufferSizeInMB(), -1);
         assertEquals(config.getManagedLedgerDataReadPriority(), "bookkeeper-first");
         assertEquals(config.getBacklogQuotaDefaultLimitGB(), 0.05);
+        assertEquals(config.getHttpMaxRequestHeaderSize(), 1234);
         OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(config.getProperties());
         assertEquals(offloadPolicies.getManagedLedgerOffloadedReadPriority().getValue(), "bookkeeper-first");
     }
diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf
index 226b2f31a738b..bfbbfb7487c42 100644
--- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf
+++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf
@@ -24,6 +24,7 @@ brokerServicePort=6650
 brokerServicePortTls=6651
 webServicePort=8080
 webServicePortTls=4443
+httpMaxRequestHeaderSize=1234
 bindAddress=0.0.0.0
 advertisedAddress=
 clusterName="test_cluster"
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
index e978f7c1ad5b1..d9dda9823ea89 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
@@ -65,6 +65,8 @@ class AdminProxyHandler extends ProxyServlet {
 
     private static final String ORIGINAL_PRINCIPAL_HEADER = "X-Original-Principal";
 
+    public static final String INIT_PARAM_REQUEST_BUFFER_SIZE = "requestBufferSize";
+
     private static final Set functionRoutes = new HashSet<>(Arrays.asList(
         "/admin/v3/function",
         "/admin/v2/function",
@@ -140,7 +142,7 @@ protected HttpClient createHttpClient() throws ServletException {
         }
         client.setIdleTimeout(Long.parseLong(value));
 
-        value = config.getInitParameter("requestBufferSize");
+        value = config.getInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE);
         if (value != null) {
             client.setRequestBufferSize(Integer.parseInt(value));
         }
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
index a91b6e70f5b8b..52e50a7e6b87b 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
@@ -661,6 +661,18 @@ public class ProxyConfiguration implements PulsarConfiguration {
     )
     private int httpOutputBufferSize = 32 * 1024;
 
+    @FieldContext(
+        minValue = 1,
+        category = CATEGORY_HTTP,
+        doc = """
+                The maximum size in bytes of the request header.
+                Larger headers will allow for more and/or larger cookies plus larger form content encoded in a URL.
+                However, larger headers consume more memory and can make a server more vulnerable to denial of service
+                attacks.
+              """
+    )
+    private int httpMaxRequestHeaderSize = 8 * 1024;
+
     @FieldContext(
             minValue = 1,
             category = CATEGORY_HTTP,
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java
index 70799aa48f259..1ca8dc93ebf9e 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.proxy.server;
 
+import static org.apache.pulsar.proxy.server.AdminProxyHandler.INIT_PARAM_REQUEST_BUFFER_SIZE;
 import io.prometheus.client.jetty.JettyStatisticsCollector;
 import java.io.IOException;
 import java.net.URI;
@@ -93,6 +94,7 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication
 
         HttpConfiguration httpConfig = new HttpConfiguration();
         httpConfig.setOutputBufferSize(config.getHttpOutputBufferSize());
+        httpConfig.setRequestHeaderSize(config.getHttpMaxRequestHeaderSize());
 
         if (config.getWebServicePort().isPresent()) {
             this.externalServicePort = config.getWebServicePort().get();
@@ -131,7 +133,7 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication
                             config.getWebServiceTlsProtocols(),
                             config.getTlsCertRefreshCheckDurationSec());
                 }
-                connectorTls = new ServerConnector(server, sslCtxFactory);
+                connectorTls = new ServerConnector(server, sslCtxFactory, new HttpConnectionFactory(httpConfig));
                 connectorTls.setPort(config.getWebServicePortTls().get());
                 connectorTls.setHost(config.getBindAddress());
                 connectors.add(connectorTls);
@@ -195,6 +197,8 @@ public void addServlet(String basePath, ServletHolder servletHolder, List> attributes, boolean requireAuthentication) {
+        popularServletParams(servletHolder, config);
+
         Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst();
         if (existingPath.isPresent()) {
             throw new IllegalArgumentException(
@@ -214,6 +218,19 @@ public void addServlet(String basePath, ServletHolder servletHolder,
         handlers.add(context);
     }
 
+    private static void popularServletParams(ServletHolder servletHolder, ProxyConfiguration config) {
+        int requestBufferSize = -1;
+        try {
+            requestBufferSize = Integer.parseInt(servletHolder.getInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE));
+        } catch (NumberFormatException nfe){
+            log.warn("The init-param {} is invalidated, because it is not a number", INIT_PARAM_REQUEST_BUFFER_SIZE);
+        }
+        if (requestBufferSize > 0 || config.getHttpMaxRequestHeaderSize() > 0) {
+            int v = Math.max(requestBufferSize, config.getHttpMaxRequestHeaderSize());
+            servletHolder.setInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE, String.valueOf(v));
+        }
+    }
+
     public void addRestResource(String basePath, String attribute, Object attributeValue, Class resourceClass) {
         ResourceConfig config = new ResourceConfig();
         config.register(resourceClass);
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java
index 7abee46aea61d..de168284ab47f 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java
@@ -75,6 +75,7 @@ protected void setup() throws Exception {
         conf.setProxyRoles(ImmutableSet.of("proxy"));
         conf.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName()));
         conf.setNumExecutorThreadPoolSize(5);
+        conf.setHttpMaxRequestHeaderSize(20000);
 
         super.internalSetup();
 
@@ -87,6 +88,7 @@ protected void setup() throws Exception {
         proxyConfig.setWebServicePort(Optional.of(0));
         proxyConfig.setWebServicePortTls(Optional.of(0));
         proxyConfig.setTlsEnabledWithBroker(true);
+        proxyConfig.setHttpMaxRequestHeaderSize(20000);
 
         // enable tls and auth&auth at proxy
         proxyConfig.setTlsCertificateFilePath(getTlsFile("broker.cert"));
@@ -185,4 +187,30 @@ public void testAuthenticatedProxyAsNonAdmin() throws Exception {
             Assert.assertEquals(ImmutableSet.of("tenant1/ns1"), user1Admin.namespaces().getNamespaces("tenant1"));
         }
     }
+
+    @Test
+    public void testAuthenticatedRequestWithLongUri() throws Exception {
+        PulsarAdmin user1Admin = getAdminClient("user1");
+        PulsarAdmin brokerAdmin = getDirectToBrokerAdminClient("admin");
+        StringBuilder longTenant = new StringBuilder("tenant");
+        for (int i = 10 * 1024; i > 0; i = i - 4){
+            longTenant.append("_abc");
+        }
+        try {
+            brokerAdmin.namespaces().getNamespaces(longTenant.toString());
+            Assert.fail("expect error: Tenant not found");
+        } catch (Exception ex){
+            Assert.assertTrue(ex instanceof PulsarAdminException);
+            PulsarAdminException pulsarAdminException = (PulsarAdminException) ex;
+            Assert.assertEquals(pulsarAdminException.getStatusCode(), 404);
+        }
+        try {
+            user1Admin.namespaces().getNamespaces(longTenant.toString());
+            Assert.fail("expect error: Tenant not found");
+        } catch (Exception ex){
+            Assert.assertTrue(ex instanceof PulsarAdminException);
+            PulsarAdminException pulsarAdminException = (PulsarAdminException) ex;
+            Assert.assertEquals(pulsarAdminException.getStatusCode(), 404);
+        }
+    }
 }
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java
index ac1078a1a4596..97a73c20b60d0 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java
@@ -68,6 +68,7 @@ public void testBackwardCompatibility() throws IOException {
         try (PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)))) {
             printWriter.println("zookeeperSessionTimeoutMs=60");
             printWriter.println("zooKeeperCacheExpirySeconds=500");
+            printWriter.println("httpMaxRequestHeaderSize=1234");
         }
         testConfigFile.deleteOnExit();
         InputStream stream = new FileInputStream(testConfigFile);
@@ -75,6 +76,7 @@ public void testBackwardCompatibility() throws IOException {
         stream.close();
         assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 60);
         assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 500);
+        assertEquals(serviceConfig.getHttpMaxRequestHeaderSize(), 1234);
 
         testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
         if (testConfigFile.exists()) {
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java
index 4b69d72e46c92..246dd9f85e319 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java
@@ -44,8 +44,12 @@
 import org.apache.pulsar.metadata.impl.ZKMetadataStore;
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
@@ -66,6 +70,7 @@ public class ProxyIsAHttpProxyTest extends MockedPulsarServiceBaseTest {
 
     private Server backingServer1;
     private Server backingServer2;
+    private Server backingServer3;
     private PulsarResources resource;
     private Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
 
@@ -85,6 +90,15 @@ protected void setup() throws Exception {
         backingServer2 = new Server(0);
         backingServer2.setHandler(newHandler("server2"));
         backingServer2.start();
+
+        backingServer3 = new Server();
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        httpConfig.setRequestHeaderSize(20000);
+        ServerConnector connector = new ServerConnector(backingServer3, new HttpConnectionFactory(httpConfig));
+        connector.setPort(0);
+        backingServer3.setConnectors(new Connector[]{connector});
+        backingServer3.setHandler(newHandler("server3"));
+        backingServer3.start();
     }
 
     private static AbstractHandler newHandler(String text) {
@@ -96,7 +110,9 @@ public void handle(String target, Request baseRequest,
                 response.setContentType("text/plain;charset=utf-8");
                 response.setStatus(HttpServletResponse.SC_OK);
                 baseRequest.setHandled(true);
-                response.getWriter().println(String.format("%s,%s", text, request.getRequestURI()));
+                String uri = request.getRequestURI();
+                response.getWriter().println(String.format("%s,%s", text,
+                        uri.substring(0, uri.length() > 1024 ? 1024 : uri.length())));
             }
         };
     }
@@ -331,6 +347,47 @@ public void testLongPath() throws Exception {
         }
     }
 
+    @Test
+    public void testLongUri() throws Exception {
+        Properties props = new Properties();
+        props.setProperty("httpReverseProxy.3.path", "/service3");
+        props.setProperty("httpReverseProxy.3.proxyTo", backingServer3.getURI().toString());
+        props.setProperty("servicePort", "0");
+        props.setProperty("webServicePort", "0");
+
+        ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class);
+        AuthenticationService authService = new AuthenticationService(
+                PulsarConfigurationLoader.convertFrom(proxyConfig));
+
+        StringBuilder longUri = new StringBuilder("/service3/tp");
+        for (int i = 10 * 1024; i > 0; i = i - 11){
+            longUri.append("_sub1_RETRY");
+        }
+
+        WebServer webServerMaxUriLen8k = new WebServer(proxyConfig, authService);
+        ProxyServiceStarter.addWebServerHandlers(webServerMaxUriLen8k, proxyConfig, null,
+                new BrokerDiscoveryProvider(proxyConfig, resource));
+        webServerMaxUriLen8k.start();
+        try {
+            Response r = client.target(webServerMaxUriLen8k.getServiceUri()).path(longUri.toString()).request().get();
+            Assert.assertEquals(r.getStatus(), Response.Status.REQUEST_URI_TOO_LONG.getStatusCode());
+        } finally {
+            webServerMaxUriLen8k.stop();
+        }
+
+        proxyConfig.setHttpMaxRequestHeaderSize(12 * 1024);
+        WebServer webServerMaxUriLen12k = new WebServer(proxyConfig, authService);
+        ProxyServiceStarter.addWebServerHandlers(webServerMaxUriLen12k, proxyConfig, null,
+                new BrokerDiscoveryProvider(proxyConfig, resource));
+        webServerMaxUriLen12k.start();
+        try {
+            Response r = client.target(webServerMaxUriLen12k.getServiceUri()).path(longUri.toString()).request().get();
+            Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
+        } finally {
+            webServerMaxUriLen12k.stop();
+        }
+    }
+
     @Test
     public void testPathEndsInSlash() throws Exception {
         Properties props = new Properties();

From 81971e2d91a18ac7aac245c36ce082cd3b8176ad Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 11 Apr 2023 14:50:08 +0800
Subject: [PATCH 169/404] [fix] [doc] Fix outdated java-doc of rate limiter
 (#19014)

### Motivation
After PR #8611, the acquiring permits can be larger than configured msg-rate if used by subscribing. But doc was not updated in time.

### Modifications

fix the outdated doc
---
 .../pulsar/broker/service/persistent/DispatchRateLimiter.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java
index 5b8987580c412..b1e4803548414 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java
@@ -99,10 +99,8 @@ public long getAvailableDispatchRateLimitOnByte() {
      */
     public boolean tryDispatchPermit(long msgPermits, long bytePermits) {
         boolean acquiredMsgPermit = msgPermits <= 0 || dispatchRateLimiterOnMessage == null
-        // acquiring permits must be < configured msg-rate;
                 || dispatchRateLimiterOnMessage.tryAcquire(msgPermits);
         boolean acquiredBytePermit = bytePermits <= 0 || dispatchRateLimiterOnByte == null
-        // acquiring permits must be < configured msg-rate;
                 || dispatchRateLimiterOnByte.tryAcquire(bytePermits);
         return acquiredMsgPermit && acquiredBytePermit;
     }

From 36579dd0b656034a7a68341c181a52ccb042439a Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 11 Apr 2023 02:43:31 -0500
Subject: [PATCH 170/404] [improve][proxy] Only create ConnectionPool when
 needed (#20062)

---
 .../pulsar/proxy/server/ProxyConnection.java  | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 36b3b4c6038fb..f03aa59619fd8 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -81,7 +81,7 @@
  */
 public class ProxyConnection extends PulsarHandler {
     private static final Logger LOG = LoggerFactory.getLogger(ProxyConnection.class);
-    // ConnectionPool is used by the proxy to issue lookup requests
+    // ConnectionPool is used by the proxy to issue lookup requests. It is null when doing direct broker proxying.
     private ConnectionPool connectionPool;
     private final AtomicLong requestIdGenerator =
             new AtomicLong(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE / 2));
@@ -313,24 +313,7 @@ protected static boolean isTlsChannel(Channel channel) {
     }
 
     private synchronized void completeConnect() throws PulsarClientException {
-        Supplier clientCnxSupplier;
-        if (service.getConfiguration().isAuthenticationEnabled()) {
-            clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole,
-                    clientAuthData, clientAuthMethod, protocolVersionToAdvertise,
-                    service.getConfiguration().isForwardAuthorizationCredentials(), this);
-        } else {
-            clientCnxSupplier = () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise);
-        }
-
-        if (this.connectionPool == null) {
-            this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(),
-                    clientCnxSupplier,
-                    Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next())));
-        } else {
-            LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}",
-                    remoteAddress, state, clientAuthRole);
-        }
-
+        checkArgument(state == State.Connecting);
         LOG.info("[{}] complete connection, init proxy handler. authenticated with {} role {}, hasProxyToBrokerUrl: {}",
                 remoteAddress, authMethod, clientAuthRole, hasProxyToBrokerUrl);
         if (hasProxyToBrokerUrl) {
@@ -371,8 +354,26 @@ private synchronized void completeConnect() throws PulsarClientException {
                     });
         } else {
             // Client is doing a lookup, we can consider the handshake complete
-            // and we'll take care of just topics and
-            // partitions metadata lookups
+            // and we'll take care of just topics and partitions metadata lookups
+            Supplier clientCnxSupplier;
+            if (service.getConfiguration().isAuthenticationEnabled()) {
+                clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole,
+                        clientAuthData, clientAuthMethod, protocolVersionToAdvertise,
+                        service.getConfiguration().isForwardAuthorizationCredentials(), this);
+            } else {
+                clientCnxSupplier =
+                        () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise);
+            }
+
+            if (this.connectionPool == null) {
+                this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(),
+                        clientCnxSupplier,
+                        Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next())));
+            } else {
+                LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}",
+                        remoteAddress, state, clientAuthRole);
+            }
+
             state = State.ProxyLookupRequests;
             lookupProxyHandler = service.newLookupProxyHandler(this);
             final ByteBuf msg = Commands.newConnected(protocolVersionToAdvertise, false);
@@ -452,7 +453,7 @@ protected void authChallengeSuccessCallback(AuthData authChallenge) {
                 }
 
                 // First connection
-                if (this.connectionPool == null || state == State.Connecting) {
+                if (state == State.Connecting) {
                     // authentication has completed, will send newConnected command.
                     completeConnect();
                 }

From 222e69a53ffc098115251253a2bc4fe8542ce55b Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 11 Apr 2023 01:59:04 -0700
Subject: [PATCH 171/404] [improve][broker] PIP-192 Improve TransferSheder for
 overload outlier for large clusters (#20059)

---
 .../extensions/scheduler/TransferShedder.java | 69 +++++++++++-----
 .../scheduler/TransferShedderTest.java        | 78 +++++++++++++++++++
 2 files changed, 126 insertions(+), 21 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 98e05296d605a..07d521a28afa7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -291,6 +291,10 @@ String peekMinBroker() {
             return brokersSortedByLoad.get(minBrokerIndex).getKey();
         }
 
+        String peekMaxBroker() {
+            return brokersSortedByLoad.get(maxBrokerIndex).getKey();
+        }
+
         String pollMaxBroker() {
             return brokersSortedByLoad.get(maxBrokerIndex--).getKey();
         }
@@ -358,7 +362,9 @@ public Set findBundlesForUnloading(LoadManagerContext context,
 
             final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd();
             boolean transfer = conf.isLoadBalancerTransferEnabled();
-            if (stats.std() > targetStd || isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
+            if (stats.std() > targetStd
+                    || isUnderLoaded(context, stats.peekMinBroker(), stats.avg)
+                    || isOverLoaded(context, stats.peekMaxBroker(), stats.avg)) {
                 unloadConditionHitCount++;
             } else {
                 unloadConditionHitCount = 0;
@@ -383,28 +389,36 @@ public Set findBundlesForUnloading(LoadManagerContext context,
                     break;
                 }
                 UnloadDecision.Reason reason;
-                if (stats.std() <= targetStd) {
-                    if (!isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
-                        if (debugMode) {
-                            log.info(CANNOT_CONTINUE_UNLOAD_MSG
-                                            + "The overall cluster load meets the target, std:{} <= targetStd:{},"
-                                            + " and minBroker:{} is not underloaded.",
-                                    stats.std(), targetStd, stats.peekMinBroker());
-                        }
-                        break;
-                    } else {
-                        reason = Underloaded;
-                        if (debugMode) {
-                            log.info(String.format("broker:%s is underloaded:%s although "
-                                            + "load std:%.2f <= targetStd:%.2f. "
-                                            + "Continuing unload for this underloaded broker.",
-                                    stats.peekMinBroker(),
-                                    context.brokerLoadDataStore().get(stats.peekMinBroker()).get(),
-                                    stats.std(), targetStd));
-                        }
+                if (stats.std() > targetStd) {
+                    reason = Overloaded;
+                } else if (isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
+                    reason = Underloaded;
+                    if (debugMode) {
+                        log.info(String.format("broker:%s is underloaded:%s although "
+                                        + "load std:%.2f <= targetStd:%.2f. "
+                                        + "Continuing unload for this underloaded broker.",
+                                stats.peekMinBroker(),
+                                context.brokerLoadDataStore().get(stats.peekMinBroker()).get(),
+                                stats.std(), targetStd));
                     }
-                } else {
+                } else if (isOverLoaded(context, stats.peekMaxBroker(), stats.avg)) {
                     reason = Overloaded;
+                    if (debugMode) {
+                        log.info(String.format("broker:%s is overloaded:%s although "
+                                        + "load std:%.2f <= targetStd:%.2f. "
+                                        + "Continuing unload for this overloaded broker.",
+                                stats.peekMaxBroker(),
+                                context.brokerLoadDataStore().get(stats.peekMaxBroker()).get(),
+                                stats.std(), targetStd));
+                    }
+                } else {
+                    if (debugMode) {
+                        log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                        + "The overall cluster load meets the target, std:{} <= targetStd:{}."
+                                        + "minBroker:{} is not underloaded. maxBroker:{} is not overloaded.",
+                                stats.std(), targetStd, stats.peekMinBroker(), stats.peekMaxBroker());
+                    }
+                    break;
                 }
 
                 String maxBroker = stats.pollMaxBroker();
@@ -671,6 +685,19 @@ private boolean isUnderLoaded(LoadManagerContext context, String broker, double
                 context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2));
     }
 
+    private boolean isOverLoaded(LoadManagerContext context, String broker, double avgLoad) {
+        var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker);
+        if (brokerLoadDataOptional.isEmpty()) {
+            return false;
+        }
+        var conf = context.brokerConfiguration();
+        var overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0;
+        var targetStd = conf.getLoadBalancerBrokerLoadTargetStd();
+        var brokerLoadData = brokerLoadDataOptional.get();
+        var load = brokerLoadData.getWeightedMaxEMA();
+        return load > overloadThreshold && load > avgLoad + targetStd;
+    }
+
 
     private boolean isTransferable(LoadManagerContext context,
                                    Map availableBrokers,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 3279cb4e475d3..d673722d6fb44 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -200,6 +200,54 @@ public LoadManagerContext setupContext(int clusterSize) {
         return ctx;
     }
 
+    public LoadManagerContext setupContextLoadSkewedOverload(int clusterSize) {
+        var ctx = getContext();
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+
+        int i = 0;
+        for (; i < clusterSize-1; i++) {
+            int brokerLoad = 1;
+            topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
+                    300_000, 700_000));
+            brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
+        }
+        int brokerLoad = 100;
+        topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
+                30_000_000, 70_000_000));
+        brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
+
+        return ctx;
+    }
+
+    public LoadManagerContext setupContextLoadSkewedUnderload(int clusterSize) {
+        var ctx = getContext();
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+
+        int i = 0;
+        for (; i < clusterSize-2; i++) {
+            int brokerLoad = 98;
+            topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
+                    30_000_000, 70_000_000));
+            brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
+        }
+
+        int brokerLoad = 99;
+        topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
+                30_000_000, 70_000_000));
+        brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
+        i++;
+
+        brokerLoad = 1;
+        topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
+                300_000, 700_000));
+        brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
+        return ctx;
+    }
+
     public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load, String broker) {
         var loadData = new BrokerLoadData();
         SystemResourceUsage usage1 = new SystemResourceUsage();
@@ -1176,6 +1224,36 @@ public void testRandomLoad() throws IllegalAccessException {
         }
     }
 
+    @Test
+    public void testOverloadOutlier() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContextLoadSkewedOverload(100);
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker99", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF",
+                        Optional.of("broker52")), Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.019900000000000008);
+        assertEquals(counter.getLoadStd(), 0.09850375627355534);
+    }
+
+    @Test
+    public void testUnderloadOutlier() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContextLoadSkewedUnderload(100);
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker98", "my-tenant/my-namespace98/0x00000000_0x0FFFFFFF",
+                        Optional.of("broker99")), Success, Underloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.9704000000000005);
+        assertEquals(counter.getLoadStd(), 0.09652895938523735);
+    }
+
     @Test
     public void testRandomLoadStats() {
         int numBrokers = 10;

From dc1cdacab8eadbd3f9d207e66d4aefbca8bca920 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 11 Apr 2023 12:17:11 +0300
Subject: [PATCH 172/404] [fix][misc] Support .nar files without explicit
 directory entries (#19304)

Co-authored-by: tison 
---
 .../src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
index a71589e6d6427..9bd5bc48df819 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
@@ -122,6 +122,8 @@ private static void unpack(final File nar, final File workingDirectory) throws I
                 if (jarEntry.isDirectory()) {
                     FileUtils.ensureDirectoryExistAndCanReadAndWrite(f);
                 } else {
+                    // The directory entry might appear after the file entry
+                    FileUtils.ensureDirectoryExistAndCanReadAndWrite(f.getParentFile());
                     makeFile(jarFile.getInputStream(jarEntry), f);
                 }
             }

From cd2aa550d0fe4e72b5ff88c4f6c1c2795b3ff2cd Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 11 Apr 2023 21:38:32 +0800
Subject: [PATCH 173/404] [improve] [broker] PIP-240: new public method
 unloadSubscription in PersistentTopic (#19737)

part-1 of PIP-240: add a new method unloadSubscription( String subName ) for PersistentTopic
---
 .../service/BrokerServiceException.java       |  12 +
 .../service/persistent/PersistentTopic.java   |  48 ++++
 .../client/api/UnloadSubscriptionTest.java    | 265 ++++++++++++++++++
 3 files changed, 325 insertions(+)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java
index fd3a391bca34a..3e77588b2459f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java
@@ -160,6 +160,18 @@ public SubscriptionNotFoundException(String msg) {
         }
     }
 
+    public static class UnsupportedSubscriptionException extends BrokerServiceException {
+        public UnsupportedSubscriptionException(String msg) {
+            super(msg);
+        }
+    }
+
+    public static class SubscriptionConflictUnloadException extends BrokerServiceException {
+        public SubscriptionConflictUnloadException(String msg) {
+            super(msg);
+        }
+    }
+
     public static class SubscriptionBusyException extends BrokerServiceException {
         public SubscriptionBusyException(String msg) {
             super(msg);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 0374fc98212f2..0f5e60439810f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -49,6 +49,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BiFunction;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import lombok.Getter;
 import org.apache.bookkeeper.client.api.LedgerMetadata;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -94,12 +95,14 @@
 import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException;
 import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException;
 import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException;
+import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionConflictUnloadException;
 import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException;
+import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedSubscriptionException;
 import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.Dispatcher;
@@ -430,6 +433,51 @@ public AtomicLong getPendingWriteOps() {
         return pendingWriteOps;
     }
 
+    /**
+     * Unload a subscriber.
+     * @throws SubscriptionNotFoundException If subscription not founded.
+     * @throws UnsupportedSubscriptionException If the subscription is typed compaction.
+     * @throws SubscriptionConflictUnloadException Conflict topic-close, topic-delete, another-subscribe-unload,
+     *     cannot unload subscription now
+     */
+    public CompletableFuture unloadSubscription(@Nonnull String subName) {
+        final PersistentSubscription sub = subscriptions.get(subName);
+        if (sub == null) {
+            return CompletableFuture.failedFuture(
+                    new SubscriptionNotFoundException(String.format("Subscription %s not found", subName)));
+        }
+        if (Compactor.COMPACTION_SUBSCRIPTION.equals(sub.getName())){
+            return CompletableFuture.failedFuture(
+                    new UnsupportedSubscriptionException(String.format("Unsupported subscription: %s", subName)));
+        }
+        // Fence old subscription -> Rewind cursor -> Replace with a new subscription.
+        return sub.disconnect().thenCompose(ignore -> {
+            if (!lock.writeLock().tryLock()) {
+                return CompletableFuture.failedFuture(new SubscriptionConflictUnloadException(String.format("Conflict"
+                        + " topic-close, topic-delete, another-subscribe-unload, cannot unload subscription %s now",
+                        topic, subName)));
+            }
+            try {
+                if (isFenced) {
+                    return CompletableFuture.failedFuture(new TopicFencedException(String.format(
+                            "Topic[%s] is fenced, can not unload subscription %s now", topic, subName)));
+                }
+                if (sub != subscriptions.get(subName)) {
+                    // Another task already finished.
+                    return CompletableFuture.failedFuture(new SubscriptionConflictUnloadException(String.format(
+                            "Another unload subscriber[%s] has been finished, do not repeat call.", subName)));
+                }
+                sub.getCursor().rewind();
+                PersistentSubscription subNew = PersistentTopic.this.createPersistentSubscription(sub.getName(),
+                        sub.getCursor(), sub.isReplicated(), sub.getSubscriptionProperties());
+                subscriptions.put(subName, subNew);
+                return CompletableFuture.completedFuture(null);
+            } finally {
+                lock.writeLock().unlock();
+            }
+        });
+    }
+
     private PersistentSubscription createPersistentSubscription(String subscriptionName, ManagedCursor cursor,
             boolean replicated, Map subscriptionProperties) {
         Objects.requireNonNull(compactedTopic);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java
new file mode 100644
index 0000000000000..93d5bf30ec6b1
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import static org.apache.pulsar.client.api.SubscriptionType.Shared;
+import static org.apache.pulsar.client.api.SubscriptionType.Key_Shared;
+import static org.apache.pulsar.client.api.SubscriptionType.Failover;
+import static org.apache.pulsar.client.api.SubscriptionType.Exclusive;
+import static org.testng.Assert.assertEquals;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.impl.BatchMessageIdImpl;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.client.impl.TopicMessageIdImpl;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker-api")
+public class UnloadSubscriptionTest extends ProducerConsumerBase {
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+    }
+
+    @Override
+    protected void doInitConf() throws Exception {
+        super.doInitConf();
+        conf.setSystemTopicEnabled(false);
+        conf.setTransactionCoordinatorEnabled(false);
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @DataProvider(name = "unloadCases")
+    public Object[][] unloadCases (){
+        // [msgCount, enabledBatch, maxMsgPerBatch, subType, ackMsgCount]
+        return new Object[][]{
+                {100, false, 1, Exclusive, 0},
+                {100, false, 1, Failover, 0},
+                {100, false, 1, Shared, 0},
+                {100, false, 1, Key_Shared, 0},
+                {100, true, 5, Exclusive, 0},
+                {100, true, 5, Failover, 0},
+                {100, true, 5, Shared, 0},
+                {100, true, 5, Key_Shared, 0},
+                {100, false, 1, Exclusive, 50},
+                {100, false, 1, Failover, 50},
+                {100, false, 1, Shared, 50},
+                {100, false, 1, Key_Shared, 50},
+                {100, true, 5, Exclusive, 50},
+                {100, true, 5, Failover, 50},
+                {100, true, 5, Shared, 50},
+                {100, true, 5, Key_Shared, 50},
+        };
+    }
+
+    @Test(dataProvider = "unloadCases")
+    public void testSingleConsumer(int msgCount, boolean enabledBatch, int maxMsgPerBatch, SubscriptionType subType,
+                                   int ackMsgCount) throws Exception {
+        final String topicName = "persistent://my-property/my-ns/tp-" + UUID.randomUUID();
+        final String subName = "sub";
+        Consumer consumer = createConsumer(topicName, subName, subType);
+        ProducerAndMessageIds producerAndMessageIds =
+                createProducerAndSendMessages(topicName, msgCount, enabledBatch, maxMsgPerBatch);
+        log.info("send message-ids:{}-{}", producerAndMessageIds.messageIds.size(),
+                toString(producerAndMessageIds.messageIds));
+
+        // Receive all messages and ack some.
+        MessagesEntry messagesEntry = receiveAllMessages(consumer);
+        assertEquals(messagesEntry.messageSet.size(), msgCount);
+        if (ackMsgCount > 0){
+            LinkedHashSet ackedMessageIds = new LinkedHashSet<>();
+            Iterator messageIdIterator = messagesEntry.messageIdSet.iterator();
+            for (int i = ackMsgCount; i > 0; i--){
+                ackedMessageIds.add(messageIdIterator.next());
+            }
+            consumer.acknowledge(ackedMessageIds.stream().toList());
+            log.info("ack message-ids: {}", toString(ackedMessageIds.stream().toList()));
+        }
+
+
+        // Unload subscriber.
+        PersistentTopic persistentTopic = getPersistentTopic(topicName);
+        persistentTopic.unloadSubscription(subName);
+        // Receive all messages for the second time.
+        MessagesEntry messagesEntryForTheSecondTime = receiveAllMessages(consumer);
+        log.info("received message-ids for the second time: {}",
+                toString(messagesEntryForTheSecondTime.messageIdSet.stream().toList()));
+        assertEquals(messagesEntryForTheSecondTime.messageSet.size(), msgCount - ackMsgCount);
+
+        // cleanup.
+        producerAndMessageIds.producer.close();
+        consumer.close();
+        admin.topics().delete(topicName);
+    }
+
+    @Test(dataProvider = "unloadCases")
+    public void testMultiConsumer(int msgCount, boolean enabledBatch, int maxMsgPerBatch, SubscriptionType subType,
+                                  int ackMsgCount) throws Exception {
+        if (subType == Exclusive){
+            return;
+        }
+        final String topicName = "persistent://my-property/my-ns/tp-" + UUID.randomUUID();
+        final String subName = "sub";
+        Consumer consumer1 = createConsumer(topicName, subName, subType);
+        Consumer consumer2 = createConsumer(topicName, subName, subType);
+        ProducerAndMessageIds producerAndMessageIds =
+                createProducerAndSendMessages(topicName, msgCount, enabledBatch, maxMsgPerBatch);
+        log.info("send message-ids:{}-{}", producerAndMessageIds.messageIds.size(),
+                toString(producerAndMessageIds.messageIds));
+
+        // Receive all messages and ack some.
+        MessagesEntry messagesEntry1 = receiveAllMessages(consumer1);
+        MessagesEntry messagesEntry2 = receiveAllMessages(consumer2);
+        LinkedHashSet allMessages = new LinkedHashSet<>();
+        allMessages.addAll(messagesEntry1.messageSet);
+        allMessages.addAll(messagesEntry2.messageSet);
+        assertEquals(allMessages.size(), msgCount);
+        if (ackMsgCount > 0){
+            LinkedHashSet allMessageIds = new LinkedHashSet<>();
+            LinkedHashSet ackedMessageIds = new LinkedHashSet<>();
+            allMessageIds.addAll(messagesEntry1.messageIdSet);
+            allMessageIds.addAll(messagesEntry2.messageIdSet);
+            Iterator messageIdIterator = allMessageIds.iterator();
+            for (int i = ackMsgCount; i > 0; i--){
+                ackedMessageIds.add(messageIdIterator.next());
+            }
+            consumer1.acknowledge(ackedMessageIds.stream().toList());
+            log.info("ack message-ids: {}", toString(ackedMessageIds.stream().toList()));
+        }
+
+        // Unload subscriber.
+        PersistentTopic persistentTopic = getPersistentTopic(topicName);
+        persistentTopic.unloadSubscription(subName);
+
+        // Receive all messages for the second time.
+        MessagesEntry messagesEntryForTheSecondTime1 = receiveAllMessages(consumer1);
+        MessagesEntry messagesEntryForTheSecondTime2 = receiveAllMessages(consumer2);
+        LinkedHashSet allMessagesForTheSecondTime = new LinkedHashSet<>();
+        allMessagesForTheSecondTime.addAll(messagesEntryForTheSecondTime1.messageSet);
+        allMessagesForTheSecondTime.addAll(messagesEntryForTheSecondTime2.messageSet);
+        LinkedHashSet allMessageIdsForTheSecondTime = new LinkedHashSet<>();
+        allMessageIdsForTheSecondTime.addAll(messagesEntry1.messageIdSet);
+        allMessageIdsForTheSecondTime.addAll(messagesEntry2.messageIdSet);
+        log.info("received message-ids for the second time: {}",
+                toString(allMessageIdsForTheSecondTime.stream().toList()));
+        assertEquals(allMessagesForTheSecondTime.size(), msgCount - ackMsgCount);
+
+        // cleanup.
+        producerAndMessageIds.producer.close();
+        consumer1.close();
+        consumer2.close();
+        admin.topics().delete(topicName);
+    }
+
+    private static String toString(List messageIds){
+        List messageIdStrings = new ArrayList<>(messageIds.size());
+        for (MessageId messageId : messageIds){
+            MessageIdImpl messageIdImpl;
+            if (messageId instanceof TopicMessageIdImpl) {
+                TopicMessageIdImpl topicMessageId = (TopicMessageIdImpl) messageId;
+                messageIdImpl = (MessageIdImpl) topicMessageId.getInnerMessageId();
+            } else {
+                messageIdImpl = (MessageIdImpl) messageId;
+            }
+            StringBuilder stringBuilder = new StringBuilder(String.valueOf(messageIdImpl.getEntryId()));
+            if (messageIdImpl instanceof BatchMessageIdImpl){
+                BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageIdImpl;
+                stringBuilder.append("_")
+                        .append(batchMessageId.getBatchIndex())
+                        .append("/")
+                        .append(batchMessageId.getBatchSize());
+            }
+            messageIdStrings.add(stringBuilder.toString());
+        }
+        return messageIdStrings.toString();
+    }
+
+    private PersistentTopic getPersistentTopic(String topicName) {
+        return (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get();
+    }
+
+    private ProducerAndMessageIds createProducerAndSendMessages(String topicName, int msgCount, boolean enabledBatch,
+                                                           int maxMsgPerBatch) throws Exception {
+        final Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .enableBatching(enabledBatch)
+                .batchingMaxMessages(maxMsgPerBatch)
+                .create();
+        ArrayList> messageIds = new ArrayList<>();
+        for (int i = 0; i < msgCount; i++) {
+            messageIds.add(producer.newMessage().key(String.valueOf(i % 10)).value(String.valueOf(i)).sendAsync());
+        }
+        FutureUtil.waitForAll(messageIds).join();
+        return new ProducerAndMessageIds(producer,
+                messageIds.stream().map(CompletableFuture::join).collect(Collectors.toList()));
+    }
+
+    private record ProducerAndMessageIds(Producer producer, List messageIds) {}
+
+    private Consumer createConsumer(String topicName, String subName, SubscriptionType subType)
+            throws Exception {
+        Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName(subName)
+                .subscriptionType(subType)
+                .isAckReceiptEnabled(true)
+                .subscribe();
+        return consumer;
+    }
+
+    private MessagesEntry receiveAllMessages(Consumer consumer) throws Exception {
+        final Set messageSet = Collections.synchronizedSet(new LinkedHashSet<>());
+        final Set messageIdSet = Collections.synchronizedSet(new LinkedHashSet<>());
+        while (true) {
+            Message msg = consumer.receive(2, TimeUnit.SECONDS);
+            if (msg == null){
+                break;
+            }
+            messageIdSet.add(msg.getMessageId());
+            messageSet.add(msg.getValue());
+        }
+        return new MessagesEntry(messageSet, messageIdSet);
+    }
+
+    private record MessagesEntry(Set messageSet, Set messageIdSet) {}
+
+}
\ No newline at end of file

From 4f503fd886cd25670b6344defb8feb1ff2a6e816 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 11 Apr 2023 21:47:02 +0800
Subject: [PATCH 174/404] [fix] [cli] Fix Broker crashed by too much memory
 usage of pulsar tools (#20031)

### Motivation
After #15868, we allow `PULSAR_MEM` & `PULSAR_GC` to be overridden in `pulsar_tool_env.sh`.

Many users set `-Xms` to `2G` or larger in `PULSAR_MEM`, this will make the tools(such as `pulsar-admin`) cost a lot of memory, and if users execute `pulsar-admin` or another tool on the machine where the Broker is deployed, the current device will not have enough memory to allocate, resulting in a broker crash.

### Modifications

When `PULSAR_MEM` is overridden  in `pulsar_tool_env.sh`, delete parameter `-Xms`
---
 conf/pulsar_tools_env.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh
index a356dbb9a28df..9d22b73905df3 100755
--- a/conf/pulsar_tools_env.sh
+++ b/conf/pulsar_tools_env.sh
@@ -42,6 +42,19 @@
 # PULSAR_GLOBAL_ZK_CONF=
 
 # Extra options to be passed to the jvm
+# Discard parameter "-Xms" of $PULSAR_MEM, which tends to be the Broker's minimum memory, to avoid using too much
+# memory by tools.
+if [ -n "$PULSAR_MEM" ]; then
+  PULSAR_MEM_ARR=("${PULSAR_MEM}")
+  PULSAR_MEM_REWRITE=""
+  for i in ${PULSAR_MEM_ARR}
+  do
+    if [ "${i:0:4}" != "-Xms" ]; then
+       PULSAR_MEM_REWRITE="$PULSAR_MEM_REWRITE $i";
+    fi
+  done
+  PULSAR_MEM=${PULSAR_MEM_REWRITE}
+fi
 PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
 
 # Garbage collection options

From f2076b44764ffbbf39e01b3873a6735a90bc3d47 Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Tue, 11 Apr 2023 22:50:04 +0800
Subject: [PATCH 175/404] [fix][test] Fix flaky
 testCreateTopicWithZombieReplicatorCursor (#20037)

---
 .../persistent/PersistentTopicTest.java       | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
index c63be7aad01cd..412b8207e34c7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
@@ -38,7 +38,9 @@
 import java.io.ByteArrayOutputStream;
 import java.lang.reflect.Field;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -52,6 +54,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedger;
@@ -83,6 +86,7 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+@Slf4j
 @Test(groups = "broker")
 public class PersistentTopicTest extends BrokerTestBase {
 
@@ -558,10 +562,10 @@ public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy)
         admin.tenants().updateTenant("prop", tenantInfo);
 
         if (topicLevelPolicy) {
-            admin.topics().setReplicationClusters(topicName, Collections.singletonList(remoteCluster));
+            admin.topics().setReplicationClusters(topicName, Arrays.asList("test", remoteCluster));
         } else {
             admin.namespaces().setNamespaceReplicationClustersAsync(
-                    namespace, Collections.singleton(remoteCluster)).get();
+                    namespace, Sets.newHashSet("test", remoteCluster)).get();
         }
 
         final PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false)
@@ -576,16 +580,27 @@ public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy)
         };
         assertEquals(getCursors.get(), Collections.singleton(conf.getReplicatorPrefix() + "." + remoteCluster));
 
+        // PersistentTopics#onPoliciesUpdate might happen in different threads, so there might be a race between two
+        // updates of the replication clusters. So here we sleep for a while to reduce the flakiness.
+        Thread.sleep(100);
+
+        // Configure the local cluster to avoid the topic being deleted in PersistentTopics#checkReplication
         if (topicLevelPolicy) {
-            admin.topics().setReplicationClusters(topicName, Collections.emptyList());
+            admin.topics().setReplicationClusters(topicName, Collections.singletonList("test"));
         } else {
-            admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.emptySet()).get();
+            admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.singleton("test")).get();
         }
         admin.clusters().deleteCluster(remoteCluster);
         // Now the cluster and its related policy has been removed but the replicator cursor still exists
 
-        topic.initialize().get(3, TimeUnit.SECONDS);
-        Awaitility.await().atMost(3, TimeUnit.SECONDS)
-                .until(() -> !topic.getManagedLedger().getCursors().iterator().hasNext());
+        Awaitility.await().atMost(Duration.ofSeconds(10)).until(() -> {
+            log.info("Before initialize...");
+            try {
+                topic.initialize().get(3, TimeUnit.SECONDS);
+            } catch (ExecutionException e) {
+                log.warn("Failed to initialize: {}", e.getCause().getMessage());
+            }
+            return !topic.getManagedLedger().getCursors().iterator().hasNext();
+        });
     }
 }

From b136cab02756e71369b9f9e7d0c0f295566624da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Tue, 11 Apr 2023 18:43:42 +0200
Subject: [PATCH 176/404] [fix][broker] Fix leader broker log (#19987)

---
 .../java/org/apache/pulsar/broker/PulsarService.java  | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index b2eefe298b53c..aad7d32f732c5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -81,6 +81,7 @@
 import org.apache.pulsar.broker.authorization.AuthorizationService;
 import org.apache.pulsar.broker.intercept.BrokerInterceptor;
 import org.apache.pulsar.broker.intercept.BrokerInterceptors;
+import org.apache.pulsar.broker.loadbalance.LeaderBroker;
 import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils;
 import org.apache.pulsar.broker.loadbalance.LoadManager;
@@ -1133,8 +1134,14 @@ protected void startLeaderElectionService() {
                         }
                     } else {
                         if (leaderElectionService != null) {
-                            LOG.info("This broker is a follower. Current leader is {}",
-                                    leaderElectionService.getCurrentLeader());
+                            final Optional currentLeader = leaderElectionService.getCurrentLeader();
+                            if (currentLeader.isPresent()) {
+                                LOG.info("This broker is a follower. Current leader is {}",
+                                        currentLeader);
+                            } else {
+                                LOG.info("This broker is a follower. No leader has been elected yet");
+                            }
+
                         }
                         if (loadSheddingTask != null) {
                             loadSheddingTask.cancel();

From 15453964f6e6086b0c5e34736958cf749f16f5e1 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 11 Apr 2023 13:19:32 -0500
Subject: [PATCH 177/404] [feat][proxy] PIP-250: Add proxyVersion to
 CommandConnect (#19618)

PIP #19623
Relates to https://github.com/apache/pulsar/pull/19540

### Motivation

In order to get more information about connections, it is helpful for the proxy to supply its version to the broker.

### Modifications

* Add `proxy_version` field to the `CommandConnect` protobuf message
* Update proxy and broker to handle this new field

### Verifying this change

New tests are added with this PR.

### Does this pull request potentially affect one of the following parts:

- [x] The binary protocol

This will be submitted as part of a PIP.

### Documentation

- [x] `doc-not-needed`
---
 .../authorization/AuthorizationService.java   |  2 +-
 .../pulsar/broker/service/ServerCnx.java      | 40 +++++++++++++++---
 .../pulsar/broker/service/TransportCnx.java   |  1 +
 .../pulsar/broker/service/ServerCnxTest.java  | 41 +++++++++++++++++++
 .../pulsar/common/protocol/Commands.java      | 11 +++++
 pulsar-common/src/main/proto/PulsarApi.proto  |  6 ++-
 .../proxy/server/DirectProxyHandler.java      |  2 +-
 .../pulsar/proxy/server/ProxyClientCnx.java   |  3 +-
 .../pulsar/proxy/server/ProxyConnection.java  |  9 ++++
 9 files changed, 105 insertions(+), 10 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index 706eadf0ec2dc..0c61219b57a50 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -464,7 +464,7 @@ public boolean isValidOriginalPrincipal(String authenticatedPrincipal,
         }
     }
 
-    private boolean isProxyRole(String role) {
+    public boolean isProxyRole(String role) {
         return role != null && conf.getProxyRoles().contains(role);
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index bfd7c001e4cfb..01274ab4460cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -210,6 +210,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
     private int pendingSendRequest = 0;
     private final String replicatorPrefix;
     private String clientVersion = null;
+    private String proxyVersion = null;
     private int nonPersistentPendingMessages = 0;
     private final int maxNonPersistentPendingMessages;
     private String originalPrincipal = null;
@@ -320,7 +321,10 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception {
             NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg);
             return;
         }
-        log.info("New connection from {}", remoteAddress);
+        if (log.isDebugEnabled()) {
+            // Connection information is logged after a successful Connect command is processed.
+            log.debug("New connection from {}", remoteAddress);
+        }
         this.ctx = ctx;
         this.commandSender = new PulsarCommandSenderImpl(brokerInterceptor, this);
         this.service.getPulsarStats().recordConnectionCreate();
@@ -690,6 +694,15 @@ private void completeConnect(int clientProtoVersion, String clientVersion) {
                     NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg);
                     return;
                 }
+                if (proxyVersion != null && !service.getAuthorizationService().isProxyRole(authRole)) {
+                    // Only allow proxyVersion to be set when connecting with a proxy
+                    state = State.Failed;
+                    service.getPulsarStats().recordConnectionCreateFail();
+                    final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError,
+                            "Must not set proxyVersion without connecting as a ProxyRole.");
+                    NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg);
+                    return;
+                }
             }
             maybeScheduleAuthenticationCredentialsRefresh();
         }
@@ -703,6 +716,18 @@ private void completeConnect(int clientProtoVersion, String clientVersion) {
         if (isNotBlank(clientVersion)) {
             this.clientVersion = clientVersion.intern();
         }
+        if (!service.isAuthenticationEnabled()) {
+            log.info("[{}] connected with clientVersion={}, clientProtocolVersion={}, proxyVersion={}", remoteAddress,
+                    clientVersion, clientProtoVersion, proxyVersion);
+        } else if (originalPrincipal != null) {
+            log.info("[{}] connected role={} and originalAuthRole={} using authMethod={}, clientVersion={}, "
+                            + "clientProtocolVersion={}, proxyVersion={}", remoteAddress, authRole, originalPrincipal,
+                    authMethod, clientVersion, clientProtoVersion, proxyVersion);
+        } else {
+            log.info("[{}] connected with role={} using authMethod={}, clientVersion={}, clientProtocolVersion={}, "
+                            + "proxyVersion={}", remoteAddress, authRole, authMethod, clientVersion, clientProtoVersion,
+                    proxyVersion);
+        }
         if (brokerInterceptor != null) {
             brokerInterceptor.onConnectionCreated(this);
         }
@@ -761,10 +786,6 @@ public void authChallengeSuccessCallback(AuthData authChallenge,
                         authenticateOriginalData(clientProtocolVersion, clientVersion);
                     } else {
                         completeConnect(clientProtocolVersion, clientVersion);
-                        if (log.isDebugEnabled()) {
-                            log.debug("[{}] Client successfully authenticated with {} role {} and originalPrincipal {}",
-                                    remoteAddress, authMethod, this.authRole, originalPrincipal);
-                        }
                     }
                 } else {
                     // Refresh the auth data
@@ -948,6 +969,10 @@ protected void handleConnect(CommandConnect connect) {
             features.copyFrom(connect.getFeatureFlags());
         }
 
+        if (connect.hasProxyVersion()) {
+            proxyVersion = connect.getProxyVersion();
+        }
+
         if (!service.isAuthenticationEnabled()) {
             completeConnect(clientProtocolVersion, clientVersion);
             return;
@@ -3266,6 +3291,11 @@ public String getClientVersion() {
         return clientVersion;
     }
 
+    @Override
+    public String getProxyVersion() {
+        return proxyVersion;
+    }
+
     @VisibleForTesting
     void setAutoReadDisabledRateLimiting(boolean isLimiting) {
         this.autoReadDisabledRateLimiting = isLimiting;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
index f1aaca2b290fa..d267160652ae4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
@@ -27,6 +27,7 @@
 public interface TransportCnx {
 
     String getClientVersion();
+    String getProxyVersion();
 
     SocketAddress clientAddress();
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 874d43896223c..c3bab634a42c1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -297,6 +297,21 @@ public void testConnectCommandWithProtocolVersion() throws Exception {
         channel.finish();
     }
 
+    @Test(timeOut = 30000)
+    public void testConnectCommandWithProxyVersion() throws Exception {
+        resetChannel();
+        assertTrue(channel.isActive());
+        assertEquals(serverCnx.getState(), State.Start);
+
+        ByteBuf clientCommand = Commands.newConnect("none", null, 1, null, null, null, null, null,
+                "my-pulsar-proxy");
+        channel.writeInbound(clientCommand);
+
+        assertEquals(serverCnx.getState(), State.Connected);
+        assertEquals(serverCnx.getProxyVersion(), "my-pulsar-proxy");
+        channel.finish();
+    }
+
     @DataProvider(name = "clientVersions")
     public Object[][] clientVersions() {
         return new Object[][]{
@@ -512,6 +527,32 @@ public void testConnectCommandWithPassingOriginalPrincipal() throws Exception {
         channel.finish();
     }
 
+    @Test
+    public void testConnectWithNonProxyRoleAndProxyVersion() throws Exception {
+        AuthenticationService authenticationService = mock(AuthenticationService.class);
+        AuthenticationProvider authenticationProvider = new MockAuthenticationProvider();
+        String authMethodName = authenticationProvider.getAuthMethodName();
+
+        when(brokerService.getAuthenticationService()).thenReturn(authenticationService);
+        when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider);
+        svcConfig.setAuthenticationEnabled(true);
+        svcConfig.setAuthorizationEnabled(true);
+
+        resetChannel();
+        assertTrue(channel.isActive());
+        assertEquals(serverCnx.getState(), State.Start);
+
+        ByteBuf clientCommand = Commands.newConnect(authMethodName, AuthData.of("pass.pass".getBytes()),
+                1, null, null, null, null, null, "my-pulsar-proxy");
+        channel.writeInbound(clientCommand);
+        Object response = getResponse();
+        assertTrue(response instanceof CommandError);
+        assertEquals(((CommandError) response).getError(), ServerError.AuthorizationError);
+        assertEquals(serverCnx.getState(), State.Failed);
+        channel.finish();
+    }
+
+    @Test
     public void testAuthChallengePrincipalChangeFails() throws Exception {
         AuthenticationService authenticationService = mock(AuthenticationService.class);
         AuthenticationProvider authenticationProvider = new MockAlwaysExpiredAuthenticationProvider();
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
index 85c4d021fdf22..cf0cd820a6d10 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
@@ -234,11 +234,22 @@ public static ByteBuf newConnect(String authMethodName, String authData, int pro
     public static ByteBuf newConnect(String authMethodName, AuthData authData, int protocolVersion, String libVersion,
                                      String targetBroker, String originalPrincipal, AuthData originalAuthData,
                                      String originalAuthMethod) {
+        return newConnect(authMethodName, authData, protocolVersion, libVersion, targetBroker, originalPrincipal,
+                originalAuthData, originalAuthMethod, null);
+    }
+
+    public static ByteBuf newConnect(String authMethodName, AuthData authData, int protocolVersion, String libVersion,
+                                     String targetBroker, String originalPrincipal, AuthData originalAuthData,
+                                     String originalAuthMethod, String proxyVersion) {
         BaseCommand cmd = localCmd(Type.CONNECT);
         CommandConnect connect = cmd.setConnect()
                 .setClientVersion(libVersion != null ? libVersion : "Pulsar Client")
                 .setAuthMethodName(authMethodName);
 
+        if (proxyVersion != null) {
+            connect.setProxyVersion(proxyVersion);
+        }
+
         if (targetBroker != null) {
             // When connecting through a proxy, we need to specify which broker do we want to be proxied through
             connect.setProxyToBrokerUrl(targetBroker);
diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto
index d9c41eeec9740..afe193eeb7e9d 100644
--- a/pulsar-common/src/main/proto/PulsarApi.proto
+++ b/pulsar-common/src/main/proto/PulsarApi.proto
@@ -268,7 +268,7 @@ enum ProtocolVersion {
 }
 
 message CommandConnect {
-    required string client_version = 1;
+    required string client_version = 1; // The version of the client. Proxy should forward client's client_version.
     optional AuthMethod auth_method = 2; // Deprecated. Use "auth_method_name" instead.
     optional string auth_method_name = 5;
     optional bytes auth_data = 3;
@@ -291,6 +291,8 @@ message CommandConnect {
 
     // Feature flags
     optional FeatureFlags feature_flags = 10;
+
+    optional string proxy_version = 11; // Version of the proxy. Should only be forwarded by a proxy.
 }
 
 message FeatureFlags {
@@ -308,7 +310,7 @@ message CommandConnected {
 }
 
 message CommandAuthResponse {
-    optional string client_version = 1;
+    optional string client_version = 1; // The version of the client. Proxy should forward client's client_version.
     optional AuthData response = 2;
     optional int32 protocol_version = 3 [default = 0];
 }
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java
index cb93f885a1d68..d63b04b6734de 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java
@@ -327,7 +327,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception {
             ByteBuf command = Commands.newConnect(
                     authentication.getAuthMethodName(), authData, protocolVersion,
                     proxyConnection.clientVersion, null /* target broker */,
-                    originalPrincipal, clientAuthData, clientAuthMethod);
+                    originalPrincipal, clientAuthData, clientAuthMethod, PulsarVersion.getVersion());
             writeAndFlush(command);
             isTlsOutboundChannel = ProxyConnection.isTlsChannel(inboundChannel);
         }
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java
index a1994fb5af4b0..00ec0b592e463 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java
@@ -23,6 +23,7 @@
 import io.netty.channel.EventLoopGroup;
 import java.util.Arrays;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.PulsarVersion;
 import org.apache.pulsar.client.impl.ClientCnx;
 import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
 import org.apache.pulsar.common.api.AuthData;
@@ -66,7 +67,7 @@ protected ByteBuf newConnectCommand() throws Exception {
         AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA);
         return Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion,
                 proxyConnection.clientVersion, proxyToTargetBrokerAddress, clientAuthRole, clientAuthData,
-                clientAuthMethod);
+                clientAuthMethod, PulsarVersion.getVersion());
     }
 
     @Override
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index f03aa59619fd8..4d32c9dce2d49 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -497,6 +497,15 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(),
             return;
         }
 
+        if (connect.hasProxyVersion()) {
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("[{}] Client illegally provided proxyVersion.", remoteAddress);
+            }
+            state = State.Closing;
+            writeAndFlushAndClose(Commands.newError(-1, ServerError.NotAllowedError, "Must not provide proxyVersion"));
+            return;
+        }
+
         try {
             // init authn
             this.clientConf = createClientConfiguration();

From 075b625993746a6c2d72343cb969845aed2cd28a Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 11 Apr 2023 15:39:26 -0500
Subject: [PATCH 178/404] [fix][proxy] Refresh auth data if ProxyLookupRequests
 (#20067)

Fixes: https://github.com/apache/pulsar/issues/10816
PIP: #19771
Supersedes: https://github.com/apache/pulsar/pull/19026
Depends on: https://github.com/apache/pulsar/pull/20062

### Motivation

The Pulsar Proxy does not properly handle authentication data refresh when in state `ProxyLookupRequests`. The consequence is described in https://github.com/apache/pulsar/issues/10816. Essentially, the problem is that the proxy caches stale authentication data and sends it to the broker leading to connection failures.

https://github.com/apache/pulsar/pull/17831 attempted to fix the underlying problem, but it missed an important edge cases. Specifically, it missed the case that the `ConnectionPool` will have multiple connections when a lookup gets redirected. As such, the following problem exists (and is fixed by this PR):

1. Client opens connection to perform lookups.
2. Proxy connects to broker 1 to get the topic ownership info.
3. Time passes.
4. Client does an additional lookup, and this topic is on a newly created broker 2. In this case, the proxy opens a new connection with the stale client auth data.
5. Broker 2 rejects the connection because it fails with expired authentication.

### Modifications

* Remove some of the implementation from https://github.com/apache/pulsar/pull/17831. This new implementation still allows a broker to challenge the client through the proxy, but notably, it limits the number of challenges sent to the client. Further, the proxy does not challenge the client when the auth data is not expired.
* Introduce authentication refresh in the proxy so that the proxy challenges the client any time the auth data is expired.
* Update the `ProxyClientCnx` to get the `clientAuthData` from the `ProxyConnection` to ensure that it gets new authentication data.
* Add clock skew to the `AuthenticationProviderToken`. This is necessary to make some of the testing not flaky and it will also be necessary for users to configure in their clusters.

### Verifying this change

The `ProxyRefreshAuthTest` covers the existing behavior and I expanded it to cover the edge case described above.

Additionally, testing this part of the code will be much easier to test once we implement #19624.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: the relevant tests pass locally, so I am going to skip the forked tests.
---
 .../AuthenticationProviderToken.java          |  22 ++-
 .../pulsar/proxy/server/ProxyClientCnx.java   |  67 +++----
 .../proxy/server/ProxyConfiguration.java      |   7 +
 .../pulsar/proxy/server/ProxyConnection.java  | 185 +++++++++++++-----
 .../proxy/server/ProxyRefreshAuthTest.java    |  31 ++-
 5 files changed, 216 insertions(+), 96 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java
index fed5ba063fd44..67e1f39a38924 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java
@@ -73,6 +73,9 @@ public class AuthenticationProviderToken implements AuthenticationProvider {
 
     // The token audience stands for this broker. The field `tokenAudienceClaim` of a valid token, need contains this.
     static final String CONF_TOKEN_AUDIENCE = "tokenAudience";
+    // The amount of time in seconds that a token is allowed to be out of sync with the server's time when performing
+    // token validation.
+    static final String CONF_TOKEN_ALLOWED_CLOCK_SKEW_SECONDS = "tokenAllowedClockSkewSeconds";
 
     static final String TOKEN = "token";
 
@@ -101,6 +104,7 @@ public class AuthenticationProviderToken implements AuthenticationProvider {
     private String confTokenPublicAlgSettingName;
     private String confTokenAudienceClaimSettingName;
     private String confTokenAudienceSettingName;
+    private String confTokenAllowedClockSkewSecondsSettingName;
 
     @Override
     public void close() throws IOException {
@@ -125,6 +129,7 @@ public void initialize(ServiceConfiguration config) throws IOException, IllegalA
         this.confTokenPublicAlgSettingName = prefix + CONF_TOKEN_PUBLIC_ALG;
         this.confTokenAudienceClaimSettingName = prefix + CONF_TOKEN_AUDIENCE_CLAIM;
         this.confTokenAudienceSettingName = prefix + CONF_TOKEN_AUDIENCE;
+        this.confTokenAllowedClockSkewSecondsSettingName = prefix + CONF_TOKEN_ALLOWED_CLOCK_SKEW_SECONDS;
 
         // we need to fetch the algorithm before we fetch the key
         this.publicKeyAlg = getPublicKeyAlgType(config);
@@ -133,7 +138,12 @@ public void initialize(ServiceConfiguration config) throws IOException, IllegalA
         this.audienceClaim = getTokenAudienceClaim(config);
         this.audience = getTokenAudience(config);
 
-        this.parser = Jwts.parserBuilder().setSigningKey(this.validationKey).build();
+        long allowedSkew = getConfTokenAllowedClockSkewSeconds(config);
+
+        this.parser = Jwts.parserBuilder()
+                .setAllowedClockSkewSeconds(allowedSkew)
+                .setSigningKey(this.validationKey)
+                .build();
 
         if (audienceClaim != null && audience == null) {
             throw new IllegalArgumentException("Token Audience Claim [" + audienceClaim
@@ -329,6 +339,16 @@ private String getTokenAudience(ServiceConfiguration conf) throws IllegalArgumen
         }
     }
 
+    // get Token's allowed clock skew in seconds. If not configured, defaults to 0.
+    private long getConfTokenAllowedClockSkewSeconds(ServiceConfiguration conf) throws IllegalArgumentException {
+        String allowedSkewStr = (String) conf.getProperty(confTokenAllowedClockSkewSecondsSettingName);
+        if (StringUtils.isNotBlank(allowedSkewStr)) {
+            return Long.parseLong(allowedSkewStr);
+        } else {
+            return 0;
+        }
+    }
+
     private static final class TokenAuthenticationState implements AuthenticationState {
         private final AuthenticationProviderToken provider;
         private final SocketAddress remoteAddress;
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java
index 00ec0b592e463..782454022b1ed 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java
@@ -29,10 +29,11 @@
 import org.apache.pulsar.common.api.AuthData;
 import org.apache.pulsar.common.api.proto.CommandAuthChallenge;
 import org.apache.pulsar.common.protocol.Commands;
+import org.apache.pulsar.common.util.netty.NettyChannelUtil;
 
 @Slf4j
 /**
- * Channel handler for Pulsar proxy's Pulsar broker client connections.
+ * Channel handler for Pulsar proxy's Pulsar broker client connections for lookup requests.
  * 

  * Please see {@link org.apache.pulsar.common.protocol.PulsarDecoder} javadoc for important details about handle*
  * method parameter instance lifecycle.
@@ -41,15 +42,13 @@ public class ProxyClientCnx extends ClientCnx {
     private final boolean forwardClientAuthData;
     private final String clientAuthMethod;
     private final String clientAuthRole;
-    private final AuthData clientAuthData;
     private final ProxyConnection proxyConnection;
 
     public ProxyClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, String clientAuthRole,
-                          AuthData clientAuthData, String clientAuthMethod, int protocolVersion,
+                          String clientAuthMethod, int protocolVersion,
                           boolean forwardClientAuthData, ProxyConnection proxyConnection) {
         super(conf, eventLoopGroup, protocolVersion);
         this.clientAuthRole = clientAuthRole;
-        this.clientAuthData = clientAuthData;
         this.clientAuthMethod = clientAuthMethod;
         this.forwardClientAuthData = forwardClientAuthData;
         this.proxyConnection = proxyConnection;
@@ -60,9 +59,15 @@ protected ByteBuf newConnectCommand() throws Exception {
         if (log.isDebugEnabled()) {
             log.debug("New Connection opened via ProxyClientCnx with params clientAuthRole = {},"
                             + " clientAuthData = {}, clientAuthMethod = {}",
-                    clientAuthRole, clientAuthData, clientAuthMethod);
+                    clientAuthRole, proxyConnection.getClientAuthData(), clientAuthMethod);
+        }
+        AuthData clientAuthData = null;
+        if (forwardClientAuthData) {
+            // There is a chance this auth data is expired because the ProxyConnection does not do early token refresh.
+            // Based on the current design, the best option is to configure the broker to accept slightly stale
+            // authentication data.
+            clientAuthData = proxyConnection.getClientAuthData();
         }
-
         authenticationDataProvider = authentication.getAuthData(remoteHostName);
         AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA);
         return Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion,
@@ -76,43 +81,21 @@ protected void handleAuthChallenge(CommandAuthChallenge authChallenge) {
         checkArgument(authChallenge.getChallenge().hasAuthData());
 
         boolean isRefresh = Arrays.equals(AuthData.REFRESH_AUTH_DATA_BYTES, authChallenge.getChallenge().getAuthData());
-        if (!forwardClientAuthData || !isRefresh) {
-            super.handleAuthChallenge(authChallenge);
-            return;
-        }
-
-        try {
-            if (log.isDebugEnabled()) {
-                log.debug("Proxy {} request to refresh the original client authentication data for "
-                        + "the proxy client {}", proxyConnection.ctx().channel(), ctx.channel());
-            }
-
-            proxyConnection.ctx().writeAndFlush(Commands.newAuthChallenge(clientAuthMethod, AuthData.REFRESH_AUTH_DATA,
-                            protocolVersion))
-                    .addListener(writeFuture -> {
-                        if (writeFuture.isSuccess()) {
-                            if (log.isDebugEnabled()) {
-                                log.debug("Proxy {} sent the auth challenge to original client to refresh credentials "
-                                                + "with method {} for the proxy client {}",
-                                        proxyConnection.ctx().channel(), clientAuthMethod, ctx.channel());
-                            }
-                        } else {
-                            log.error("Failed to send the auth challenge to original client by the proxy {} "
-                                            + "for the proxy client {}",
-                                    proxyConnection.ctx().channel(),
-                                    ctx.channel(),
-                                    writeFuture.cause());
-                            closeWithException(writeFuture.cause());
-                        }
+        if (forwardClientAuthData && isRefresh) {
+            proxyConnection.getValidClientAuthData()
+                    .thenApplyAsync(authData -> {
+                        NettyChannelUtil.writeAndFlushWithVoidPromise(ctx,
+                                Commands.newAuthResponse(clientAuthMethod, authData, this.protocolVersion,
+                                        String.format("Pulsar-Java-v%s", PulsarVersion.getVersion())));
+                        return null;
+                        }, ctx.executor())
+                    .exceptionally(ex -> {
+                        log.warn("Failed to get valid client auth data. Closing connection.", ex);
+                        ctx.close();
+                        return null;
                     });
-
-            if (state == State.SentConnectFrame) {
-                state = State.Connecting;
-            }
-        } catch (Exception e) {
-            log.error("Failed to send the auth challenge to origin client by the proxy {} for the proxy client {}",
-                    proxyConnection.ctx().channel(), ctx.channel(), e);
-            closeWithException(e);
+        } else {
+            super.handleAuthChallenge(authChallenge);
         }
     }
 }
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
index 52e50a7e6b87b..3ecd670cbbf7a 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
@@ -364,6 +364,13 @@ public class ProxyConfiguration implements PulsarConfiguration {
             + "to take effect"
     )
     private boolean forwardAuthorizationCredentials = false;
+
+    @FieldContext(
+            category = CATEGORY_AUTHENTICATION,
+            doc = "Interval of time for checking for expired authentication credentials. Disable by setting to 0."
+    )
+    private int authenticationRefreshCheckSeconds = 60;
+
     @FieldContext(
         category = CATEGORY_AUTHENTICATION,
         doc = "Whether the '/metrics' endpoint requires authentication. Defaults to true."
diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 4d32c9dce2d49..ba9247a085dff 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -29,13 +29,17 @@
 import io.netty.handler.codec.haproxy.HAProxyMessage;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.resolver.dns.DnsAddressResolverGroup;
+import io.netty.util.concurrent.ScheduledFuture;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.nio.channels.ClosedChannelException;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
@@ -64,10 +68,12 @@
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
 import org.apache.pulsar.common.api.proto.CommandLookupTopic;
 import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata;
+import org.apache.pulsar.common.api.proto.FeatureFlags;
 import org.apache.pulsar.common.api.proto.ProtocolVersion;
 import org.apache.pulsar.common.api.proto.ServerError;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.PulsarHandler;
+import org.apache.pulsar.common.util.Runnables;
 import org.apache.pulsar.common.util.netty.NettyChannelUtil;
 import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
 import org.slf4j.Logger;
@@ -92,10 +98,16 @@ public class ProxyConnection extends PulsarHandler {
     private LookupProxyHandler lookupProxyHandler = null;
     @Getter
     private DirectProxyHandler directProxyHandler = null;
+    private ScheduledFuture authRefreshTask;
+    // When authChallengeSentTime is not Long.MAX_VALUE, it means the proxy is waiting for the client to respond
+    // to an auth challenge. When authChallengeSentTime is Long.MAX_VALUE, there are no pending auth challenges.
+    private long authChallengeSentTime = Long.MAX_VALUE;
+    private FeatureFlags features;
+    private Set> pendingBrokerAuthChallenges = null;
     private final BrokerProxyValidator brokerProxyValidator;
     private final ConnectionController connectionController;
     String clientAuthRole;
-    AuthData clientAuthData;
+    volatile AuthData clientAuthData;
     String clientAuthMethod;
     String clientVersion;
 
@@ -191,6 +203,15 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep
             directProxyHandler = null;
         }
 
+        if (authRefreshTask != null) {
+            authRefreshTask.cancel(false);
+        }
+
+        if (pendingBrokerAuthChallenges != null) {
+            pendingBrokerAuthChallenges.forEach(future -> future.cancel(true));
+            pendingBrokerAuthChallenges = null;
+        }
+
         service.getClientCnxs().remove(this);
         LOG.info("[{}] Connection closed", remoteAddress);
 
@@ -358,7 +379,7 @@ private synchronized void completeConnect() throws PulsarClientException {
             Supplier clientCnxSupplier;
             if (service.getConfiguration().isAuthenticationEnabled()) {
                 clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole,
-                        clientAuthData, clientAuthMethod, protocolVersionToAdvertise,
+                        clientAuthMethod, protocolVersionToAdvertise,
                         service.getConfiguration().isForwardAuthorizationCredentials(), this);
             } else {
                 clientCnxSupplier =
@@ -376,6 +397,15 @@ private synchronized void completeConnect() throws PulsarClientException {
 
             state = State.ProxyLookupRequests;
             lookupProxyHandler = service.newLookupProxyHandler(this);
+            if (service.getConfiguration().isAuthenticationEnabled()
+                    && service.getConfiguration().getAuthenticationRefreshCheckSeconds() > 0) {
+                authRefreshTask = ctx.executor().scheduleAtFixedRate(
+                        Runnables.catchingAndLoggingThrowables(
+                                this::refreshAuthenticationCredentialsAndCloseIfTooExpired),
+                        service.getConfiguration().getAuthenticationRefreshCheckSeconds(),
+                        service.getConfiguration().getAuthenticationRefreshCheckSeconds(),
+                        TimeUnit.SECONDS);
+            }
             final ByteBuf msg = Commands.newConnected(protocolVersionToAdvertise, false);
             writeAndFlush(msg);
         }
@@ -472,6 +502,61 @@ protected void authChallengeSuccessCallback(AuthData authChallenge) {
         }
     }
 
+    private void refreshAuthenticationCredentialsAndCloseIfTooExpired() {
+        assert ctx.executor().inEventLoop();
+        if (state != State.ProxyLookupRequests) {
+            // Happens when an exception is thrown that causes this connection to close.
+            return;
+        } else if (!authState.isExpired()) {
+            // Credentials are still valid. Nothing to do at this point
+            return;
+        }
+
+        if (System.nanoTime() - authChallengeSentTime
+                > TimeUnit.SECONDS.toNanos(service.getConfiguration().getAuthenticationRefreshCheckSeconds())) {
+            LOG.warn("[{}] Closing connection after timeout on refreshing auth credentials", remoteAddress);
+            ctx.close();
+        }
+
+        maybeSendAuthChallenge();
+    }
+
+    private void maybeSendAuthChallenge() {
+        assert ctx.executor().inEventLoop();
+
+        if (!supportsAuthenticationRefresh()) {
+            LOG.warn("[{}] Closing connection because client doesn't support auth credentials refresh", remoteAddress);
+            ctx.close();
+            return;
+        } else if (authChallengeSentTime != Long.MAX_VALUE) {
+            // If the proxy sent a refresh but hasn't yet heard back, do not send another challenge.
+            return;
+        } else if (service.getConfiguration().getAuthenticationRefreshCheckSeconds() < 1) {
+            // Without the refresh check enabled, there is no way to guarantee the ProxyConnection will close
+            // this connection if the client fails to respond to the auth challenge with valid auth data.
+            // The cost is minimal since the client can recreate the connection. This logic prevents a leak.
+            LOG.warn("[{}] Closing connection because auth credentials refresh is disabled", remoteAddress);
+            ctx.close();
+            return;
+        }
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("[{}] Refreshing authentication credentials", remoteAddress);
+        }
+        try {
+            AuthData challenge = authState.refreshAuthentication();
+            writeAndFlush(Commands.newAuthChallenge(authMethod, challenge, protocolVersionToAdvertise));
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("[{}] Sent auth challenge to client to refresh credentials with method: {}.",
+                        remoteAddress, authMethod);
+            }
+            authChallengeSentTime = System.nanoTime();
+        } catch (AuthenticationException e) {
+            LOG.warn("[{}] Failed to refresh authentication: {}", remoteAddress, e);
+            ctx.close();
+        }
+    }
+
     @Override
     protected void handleConnect(CommandConnect connect) {
         checkArgument(state == State.Init);
@@ -481,6 +566,10 @@ protected void handleConnect(CommandConnect connect) {
         this.protocolVersionToAdvertise = getProtocolVersionToAdvertise(connect);
         this.proxyToBrokerUrl = connect.hasProxyToBrokerUrl() ? connect.getProxyToBrokerUrl() : "null";
         this.clientVersion = connect.getClientVersion();
+        features = new FeatureFlags();
+        if (connect.hasFeatureFlags()) {
+            features.copyFrom(connect.getFeatureFlags());
+        }
 
         if (LOG.isDebugEnabled()) {
             LOG.debug("Received CONNECT from {} proxyToBroker={}", remoteAddress, proxyToBrokerUrl);
@@ -574,53 +663,25 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) {
         }
 
         try {
+            // Reset the auth challenge sent time to indicate we are not waiting on a client response.
+            authChallengeSentTime = Long.MAX_VALUE;
             AuthData clientData = AuthData.of(authResponse.getResponse().getAuthData());
+            // Authenticate the client's auth data and send to the broker concurrently
+            // Note: this implementation relies on the current weakness that prevents multi-stage authentication
+            // from working when forwardAuthorizationCredentials is enabled. Here is an issue to fix the protocol:
+            // https://github.com/apache/pulsar/issues/19291.
             doAuthentication(clientData);
-            if (service.getConfiguration().isForwardAuthorizationCredentials()
-                    && connectionPool != null && state == State.ProxyLookupRequests) {
-                connectionPool.getConnections().forEach(toBrokerCnxFuture -> {
-                    String clientVersion;
-                    if (authResponse.hasClientVersion()) {
-                        clientVersion = authResponse.getClientVersion();
-                    } else {
-                        clientVersion = this.clientVersion;
-                    }
-                    int protocolVersion;
-                    if (authResponse.hasProtocolVersion()) {
-                        protocolVersion = authResponse.getProtocolVersion();
-                    } else {
-                        protocolVersion = Commands.getCurrentProtocolVersion();
+            if (service.getConfiguration().isForwardAuthorizationCredentials()) {
+                // Update the clientAuthData to be able to initialize future ProxyClientCnx.
+                this.clientAuthData = clientData;
+                // We only have pendingBrokerAuthChallenges when forwardAuthorizationCredentials is enabled.
+                if (pendingBrokerAuthChallenges != null && !pendingBrokerAuthChallenges.isEmpty()) {
+                    // Send auth data to pending challenges from the broker
+                    for (CompletableFuture challenge : pendingBrokerAuthChallenges) {
+                        challenge.complete(clientData);
                     }
-
-                    ByteBuf cmd =
-                            Commands.newAuthResponse(clientAuthMethod, clientData, protocolVersion, clientVersion);
-                    toBrokerCnxFuture.thenAccept(toBrokerCnx -> toBrokerCnx.ctx().writeAndFlush(cmd)
-                                    .addListener(writeFuture -> {
-                                        if (writeFuture.isSuccess()) {
-                                            if (LOG.isDebugEnabled()) {
-                                                LOG.debug("{} authentication is refreshed successfully by {}, "
-                                                                + "auth method: {} ",
-                                                        toBrokerCnx.ctx().channel(), ctx.channel(), clientAuthMethod);
-                                            }
-                                        } else {
-                                            LOG.error("Failed to forward the auth response "
-                                                            + "from the proxy to the broker through the proxy client, "
-                                                            + "proxy: {}, proxy client: {}",
-                                                    ctx.channel(),
-                                                    toBrokerCnx.ctx().channel(),
-                                                    writeFuture.cause());
-                                            toBrokerCnx.ctx().channel().pipeline()
-                                                    .fireExceptionCaught(writeFuture.cause());
-                                        }
-                                    }))
-                            .whenComplete((__, ex) -> {
-                                if (ex != null) {
-                                    LOG.error("Failed to forward the auth response from the proxy to "
-                                                    + "the broker through the proxy client, proxy: {}",
-                                            ctx().channel(), ex);
-                                }
-                            });
-                });
+                    pendingBrokerAuthChallenges.clear();
+                }
             }
         } catch (Exception e) {
             String errorMsg = "Unable to handleAuthResponse";
@@ -769,4 +830,36 @@ private void writeAndFlush(ByteBuf cmd) {
     private void writeAndFlushAndClose(ByteBuf cmd) {
         NettyChannelUtil.writeAndFlushWithClosePromise(ctx, cmd);
     }
+
+    boolean supportsAuthenticationRefresh() {
+        return features != null && features.isSupportsAuthRefresh();
+    }
+
+    AuthData getClientAuthData() {
+        return clientAuthData;
+    }
+
+    /**
+     * Thread-safe method to retrieve unexpired client auth data. Due to inherent race conditions,
+     * the auth data may expire before it is used.
+     */
+    CompletableFuture getValidClientAuthData() {
+        final CompletableFuture clientAuthDataFuture = new CompletableFuture<>();
+        ctx().executor().execute(Runnables.catchingAndLoggingThrowables(() -> {
+            // authState is not thread safe, so this must run on the ProxyConnection's event loop.
+            if (!authState.isExpired()) {
+                clientAuthDataFuture.complete(clientAuthData);
+            } else if (state == State.ProxyLookupRequests) {
+                maybeSendAuthChallenge();
+                if (pendingBrokerAuthChallenges == null) {
+                    pendingBrokerAuthChallenges = new HashSet<>();
+                }
+                pendingBrokerAuthChallenges.add(clientAuthDataFuture);
+            } else {
+                clientAuthDataFuture.completeExceptionally(new PulsarClientException.AlreadyClosedException(
+                        "ProxyConnection is not in a valid state to get client auth data for " + remoteAddress));
+            }
+        }));
+        return clientAuthDataFuture;
+    }
 }
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java
index d14105b0b43c2..bde989fc432f9 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.proxy.server;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertTrue;
 import com.google.common.collect.Sets;
 import io.jsonwebtoken.SignatureAlgorithm;
@@ -31,13 +30,11 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import javax.crypto.SecretKey;
-import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
 import org.apache.pulsar.broker.authentication.AuthenticationService;
 import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils;
 import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerConsumerBase;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.impl.ClientCnx;
@@ -81,6 +78,9 @@ protected void doInitConf() throws Exception {
         conf.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName()));
         Properties properties = new Properties();
         properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY));
+        // The skew should be double the proxy's refresh interval to ensure the broker accepts auth data
+        // that the proxy might forward.
+        properties.setProperty("tokenAllowedClockSkewSeconds", "2");
         conf.setProperties(properties);
 
         conf.setClusterName("proxy-authorization");
@@ -108,6 +108,7 @@ protected void setup() throws Exception {
         proxyConfig.setAuthenticationEnabled(true);
         proxyConfig.setAuthorizationEnabled(false);
         proxyConfig.setForwardAuthorizationCredentials(true);
+        proxyConfig.setAuthenticationRefreshCheckSeconds(1);
         proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl());
         proxyConfig.setAdvertisedAddress(null);
 
@@ -162,14 +163,30 @@ public void testAuthDataRefresh(boolean forwardAuthData) throws Exception {
                 .authentication(authenticationToken));
 
         String topic = "persistent://my-tenant/my-ns/my-topic1";
-        @Cleanup
-        Producer ignored = spy(pulsarClient.newProducer()
-                .topic(topic).create());
 
         PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient;
+        pulsarClient.getPartitionsForTopic(topic).get();
         Set> connections = pulsarClientImpl.getCnxPool().getConnections();
 
-        Awaitility.await().during(4, SECONDS).untilAsserted(() -> {
+        Awaitility.await().during(5, SECONDS).untilAsserted(() -> {
+            pulsarClient.getPartitionsForTopic(topic).get();
+            assertTrue(connections.stream().allMatch(n -> {
+                try {
+                    ClientCnx clientCnx = n.get();
+                    long timestamp = clientCnx.getLastDisconnectedTimestamp();
+                    return timestamp == 0;
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }));
+        });
+
+        // Force all connections from proxy to broker to close and therefore require the proxy to re-authenticate with
+        // the broker. (The client doesn't lose this connection.)
+        restartBroker();
+
+        // Rerun assertion to ensure that it still works
+        Awaitility.await().during(5, SECONDS).untilAsserted(() -> {
             pulsarClient.getPartitionsForTopic(topic).get();
             assertTrue(connections.stream().allMatch(n -> {
                 try {

From 6514cddee5ef1f1ecf8a63fc78429d65cd5ffd36 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Wed, 12 Apr 2023 09:23:34 +0800
Subject: [PATCH 179/404] [improve][broker] PIP-192: Redirect the request to
 current load manager (#20058)

---
 .../loadbalance/LeaderElectionService.java    |   9 +-
 .../broker/loadbalance/NoopLoadManager.java   |   1 +
 .../extensions/BrokerRegistryImpl.java        |  15 +--
 .../extensions/ExtensibleLoadManagerImpl.java |   6 +-
 .../extensions/data/BrokerLookupData.java     |  12 ++
 .../filter/BrokerLoadManagerClassFilter.java  |  50 ++++++++
 .../extensions/manager/RedirectManager.java   | 116 ++++++++++++++++++
 .../impl/BrokerLoadManagerClassFilter.java    |  41 +++++++
 .../impl/ModularLoadManagerImpl.java          |   2 +
 .../impl/SimpleLoadManagerImpl.java           |   2 +
 .../broker/namespace/NamespaceService.java    |  57 ++++++---
 .../extensions/BrokerRegistryTest.java        |  11 +-
 .../ExtensibleLoadManagerImplTest.java        |  95 +++++++++++---
 .../extensions/data/BrokerLookupDataTest.java |   4 +-
 .../filter/BrokerFilterTestBase.java          |   8 +-
 .../BrokerIsolationPoliciesFilterTest.java    |   4 +-
 .../BrokerLoadManagerClassFilterTest.java     |  65 ++++++++++
 .../scheduler/TransferShedderTest.java        |   3 +-
 .../BrokerLoadManagerClassFilterTest.java     |  73 +++++++++++
 .../data/loadbalancer/ServiceLookupData.java  |   4 +
 .../data/loadbalancer/LoadReport.java         |  20 +++
 .../data/loadbalancer/LocalBrokerData.java    |  16 +++
 22 files changed, 568 insertions(+), 46 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java
index 5ef95761e4ec6..05fe4353f3e76 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java
@@ -37,7 +37,14 @@ public class LeaderElectionService implements AutoCloseable {
 
     public LeaderElectionService(CoordinationService cs, String localWebServiceAddress,
             Consumer listener) {
-        this.leaderElection = cs.getLeaderElection(LeaderBroker.class, ELECTION_ROOT, listener);
+        this(cs, localWebServiceAddress, ELECTION_ROOT, listener);
+    }
+
+    public LeaderElectionService(CoordinationService cs,
+                                 String localWebServiceAddress,
+                                 String electionRoot,
+                                 Consumer listener) {
+        this.leaderElection = cs.getLeaderElection(LeaderBroker.class, electionRoot, listener);
         this.localValue = new LeaderBroker(localWebServiceAddress);
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java
index e8c0567fd0c8c..0de2ae92db61a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java
@@ -65,6 +65,7 @@ public void start() throws PulsarServerException {
                 pulsar.getWebServiceAddressTls(),
                 pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getAdvertisedListeners());
         localData.setProtocols(pulsar.getProtocolDataToAdvertise());
+        localData.setLoadManagerClassName(this.pulsar.getConfig().getLoadManagerClassName());
         String brokerReportPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress;
 
         try {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java
index 4c2beba5d30b3..921ce35b5c65e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java
@@ -18,8 +18,8 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions;
 
+import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -51,8 +51,6 @@
 @Slf4j
 public class BrokerRegistryImpl implements BrokerRegistry {
 
-    protected static final String LOOKUP_DATA_PATH = "/loadbalance/extension/brokers";
-
     private final PulsarService pulsar;
 
     private final ServiceConfiguration conf;
@@ -94,6 +92,8 @@ public BrokerRegistryImpl(PulsarService pulsar) {
                 pulsar.getProtocolDataToAdvertise(),
                 pulsar.getConfiguration().isEnablePersistentTopics(),
                 pulsar.getConfiguration().isEnableNonPersistentTopics(),
+                conf.getLoadManagerClassName(),
+                System.currentTimeMillis(),
                 pulsar.getBrokerVersion());
         this.state = State.Init;
     }
@@ -151,7 +151,7 @@ public String getBrokerId() {
     @Override
     public CompletableFuture> getAvailableBrokersAsync() {
         this.checkState();
-        return brokerLookupDataLockManager.listLocks(LOOKUP_DATA_PATH).thenApply(Lists::newArrayList);
+        return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenApply(ArrayList::new);
     }
 
     @Override
@@ -215,7 +215,7 @@ private void handleMetadataStoreNotification(Notification t) {
                 return;
             }
             this.scheduler.submit(() -> {
-                String brokerId = t.getPath().substring(LOOKUP_DATA_PATH.length() + 1);
+                String brokerId = t.getPath().substring(LOADBALANCE_BROKERS_ROOT.length() + 1);
                 for (BiConsumer listener : listeners) {
                     listener.accept(brokerId, t.getType());
                 }
@@ -227,12 +227,13 @@ private void handleMetadataStoreNotification(Notification t) {
 
     @VisibleForTesting
     protected static boolean isVerifiedNotification(Notification t) {
-       return t.getPath().startsWith(LOOKUP_DATA_PATH + "/") && t.getPath().length() > LOOKUP_DATA_PATH.length() + 1;
+       return t.getPath().startsWith(LOADBALANCE_BROKERS_ROOT + "/")
+               && t.getPath().length() > LOADBALANCE_BROKERS_ROOT.length() + 1;
     }
 
     @VisibleForTesting
     protected static String keyPath(String brokerId) {
-        return String.format("%s/%s", LOOKUP_DATA_PATH, brokerId);
+        return String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, brokerId);
     }
 
     private void checkState() throws IllegalStateException {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index e054a1bc662c7..4ebf537f7a8a8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -52,6 +52,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerLoadManagerClassFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
@@ -106,6 +107,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000;
 
+    private static final String ELECTION_ROOT = "/loadbalance/extension/leader";
+
     private PulsarService pulsar;
 
     private ServiceConfiguration conf;
@@ -187,6 +190,7 @@ public enum Role {
      */
     public ExtensibleLoadManagerImpl() {
         this.brokerFilterPipeline = new ArrayList<>();
+        this.brokerFilterPipeline.add(new BrokerLoadManagerClassFilter());
         this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter());
         this.brokerFilterPipeline.add(new BrokerVersionFilter());
         // TODO: Make brokerSelectionStrategy configurable.
@@ -215,7 +219,7 @@ public void start() throws PulsarServerException {
         }
         this.brokerRegistry = new BrokerRegistryImpl(pulsar);
         this.leaderElectionService = new LeaderElectionService(
-                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
+                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), ELECTION_ROOT,
                 state -> {
                     pulsar.getLoadManagerExecutor().execute(() -> {
                         if (state == LeaderElectionState.Leading) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
index 4c9e503129e66..41f5b18e321e8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
@@ -36,6 +36,8 @@ public record BrokerLookupData (String webServiceUrl,
                                 Map protocols,
                                 boolean persistentTopicsEnabled,
                                 boolean nonPersistentTopicsEnabled,
+                                String loadManagerClassName,
+                                long startTimestamp,
                                 String brokerVersion) implements ServiceLookupData {
     @Override
     public String getWebServiceUrl() {
@@ -67,6 +69,16 @@ public Optional getProtocol(String protocol) {
         return Optional.ofNullable(this.protocols().get(protocol));
     }
 
+    @Override
+    public String getLoadManagerClassName() {
+        return this.loadManagerClassName;
+    }
+
+    @Override
+    public long getStartTimestamp() {
+        return this.startTimestamp;
+    }
+
     public LookupResult toLookupResult() {
         return new LookupResult(webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls,
                 LookupResult.Type.BrokerUrl, false);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
new file mode 100644
index 0000000000000..4ee28a5225a0d
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.filter;
+
+import java.util.Map;
+import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.common.naming.ServiceUnitId;
+
+public class BrokerLoadManagerClassFilter implements BrokerFilter {
+
+    public static final String FILTER_NAME = "broker_load_manager_class_filter";
+    @Override
+    public String name() {
+        return FILTER_NAME;
+    }
+
+    @Override
+    public Map filter(
+            Map brokers,
+            ServiceUnitId serviceUnit,
+            LoadManagerContext context)
+            throws BrokerFilterException {
+        if (brokers.isEmpty()) {
+            return brokers;
+        }
+        brokers.entrySet().removeIf(entry -> {
+            BrokerLookupData v = entry.getValue();
+            return !v.getLoadManagerClassName().equals(context.brokerConfiguration().getLoadManagerClassName());
+        });
+        return brokers;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
new file mode 100644
index 0000000000000..4aff77937a5b4
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.lookup.LookupResult;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.metadata.api.coordination.LockManager;
+import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
+
+@Slf4j
+public class RedirectManager {
+    private final PulsarService pulsar;
+
+    private final LockManager brokerLookupDataLockManager;
+
+
+    public RedirectManager(PulsarService pulsar) {
+        this.pulsar = pulsar;
+        this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class);
+    }
+
+    public CompletableFuture> getAvailableBrokerLookupDataAsync() {
+        return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenCompose(availableBrokers -> {
+            Map map = new ConcurrentHashMap<>();
+            List> futures = new ArrayList<>();
+            for (String brokerId : availableBrokers) {
+                futures.add(this.brokerLookupDataLockManager.readLock(
+                        String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, brokerId)).thenAccept(lookupDataOpt -> {
+                    if (lookupDataOpt.isPresent()) {
+                        map.put(brokerId, lookupDataOpt.get());
+                    } else {
+                        log.warn("Got an empty lookup data, brokerId: {}", brokerId);
+                    }
+                }));
+            }
+
+            return FutureUtil.waitForAll(futures).thenApply(__ -> map);
+        });
+    }
+
+    public CompletableFuture> findRedirectLookupResultAsync() {
+        String currentLMClassName = pulsar.getConfiguration().getLoadManagerClassName();
+        boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfig(), log);
+        return getAvailableBrokerLookupDataAsync().thenApply(lookupDataMap -> {
+            if (lookupDataMap.isEmpty()) {
+                String errorMsg = "No available broker found.";
+                log.warn(errorMsg);
+                throw new IllegalStateException(errorMsg);
+            }
+            AtomicReference latestServiceLookupData = new AtomicReference<>();
+            AtomicLong lastStartTimestamp = new AtomicLong(0L);
+            lookupDataMap.forEach((key, value) -> {
+                if (lastStartTimestamp.get() <= value.getStartTimestamp()) {
+                    lastStartTimestamp.set(value.getStartTimestamp());
+                    latestServiceLookupData.set(value);
+                }
+            });
+            if (latestServiceLookupData.get() == null) {
+                String errorMsg = "No latest service lookup data found.";
+                log.warn(errorMsg);
+                throw new IllegalStateException(errorMsg);
+            }
+            if (latestServiceLookupData.get().getLoadManagerClassName().equals(currentLMClassName)) {
+                if (debug) {
+                    log.info("We don't need to redirect, current load manager class name: {}",
+                            currentLMClassName);
+                }
+                return Optional.empty();
+            }
+            var serviceLookupDataObj = latestServiceLookupData.get();
+            var candidateBrokers = new ArrayList();
+            lookupDataMap.forEach((key, value) -> {
+                if (value.getLoadManagerClassName().equals(serviceLookupDataObj.getLoadManagerClassName())) {
+                    candidateBrokers.add(value);
+                }
+            });
+            var selectedBroker = candidateBrokers.get((int) (Math.random() * candidateBrokers.size()));
+
+            return Optional.of(new LookupResult(selectedBroker.getWebServiceUrl(),
+                    selectedBroker.getWebServiceUrlTls(),
+                    selectedBroker.getPulsarServiceUrl(),
+                    selectedBroker.getPulsarServiceUrlTls(),
+                    true));
+        });
+    }
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
new file mode 100644
index 0000000000000..5d6a56ba86960
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.impl;
+
+import java.util.Set;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.BrokerFilter;
+import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.LoadData;
+import org.apache.pulsar.policies.data.loadbalancer.BundleData;
+
+public class BrokerLoadManagerClassFilter implements BrokerFilter {
+
+    @Override
+    public void filter(Set brokers, BundleData bundleToAssign,
+                       LoadData loadData,
+                       ServiceConfiguration conf) throws BrokerFilterException {
+        loadData.getBrokerData().forEach((key, value) -> {
+            if (!value.getLocalData().getLoadManagerClassName()
+                    .equals(conf.getLoadManagerClassName())) {
+                brokers.remove(key);
+            }
+        });
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index 4756b885ff217..c6f406a7c8a5c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -267,6 +267,7 @@ public void initialize(final PulsarService pulsar) {
 
         placementStrategy = ModularLoadManagerStrategy.create(conf);
         policies = new SimpleResourceAllocationPolicies(pulsar);
+        filterPipeline.add(new BrokerLoadManagerClassFilter());
         filterPipeline.add(new BrokerVersionFilter());
 
         LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap);
@@ -960,6 +961,7 @@ public void start() throws PulsarServerException {
             // configure broker-topic mode
             localData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
             localData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
+            localData.setLoadManagerClassName(conf.getLoadManagerClassName());
 
             String lookupServiceAddress = pulsar.getLookupServiceAddress();
             brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java
index c98857b2fc44c..5e99456971147 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java
@@ -1090,6 +1090,8 @@ private LoadReport generateLoadReportForcefully() throws Exception {
                 loadReport.setSystemResourceUsage(systemResourceUsage);
                 loadReport.setBundleStats(pulsar.getBrokerService().getBundleStats());
                 loadReport.setTimestamp(System.currentTimeMillis());
+                loadReport.setLoadManagerClassName(pulsar.getConfig().getLoadManagerClassName());
+                loadReport.setStartTimestamp(System.currentTimeMillis());
 
                 final Set oldBundles = lastLoadReport.getBundles();
                 final Set newBundles = loadReport.getBundles();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index c9cd33d6d87c8..a0e2bf7534c02 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -56,6 +56,7 @@
 import org.apache.pulsar.broker.loadbalance.LoadManager;
 import org.apache.pulsar.broker.loadbalance.ResourceUnit;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.RedirectManager;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.resources.NamespaceResources;
 import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException;
@@ -136,6 +137,7 @@ public class NamespaceService implements AutoCloseable {
     private final ConcurrentOpenHashMap namespaceClients;
 
     private final List bundleOwnershipListeners;
+    private final RedirectManager redirectManager;
 
 
     private static final Counter lookupRedirects = Counter.build("pulsar_broker_lookup_redirects", "-").register();
@@ -164,6 +166,7 @@ public NamespaceService(PulsarService pulsar) {
                 ConcurrentOpenHashMap.newBuilder().build();
         this.bundleOwnershipListeners = new CopyOnWriteArrayList<>();
         this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class);
+        this.redirectManager = new RedirectManager(pulsar);
     }
 
     public void initialize() {
@@ -177,12 +180,20 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN
 
         CompletableFuture> future = getBundleAsync(topic)
                 .thenCompose(bundle -> {
-                    if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
-                        return loadManager.get().findBrokerServiceUrl(Optional.of(topic), bundle);
-                    } else {
-                        // TODO: Add unit tests cover it.
-                        return findBrokerServiceUrl(bundle, options);
-                    }
+                    // Do redirection if the cluster is in rollback or deploying.
+                    return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> {
+                        if (optResult.isPresent()) {
+                            LOG.info("[{}] Redirect lookup request to {} for topic {}",
+                                    pulsar.getSafeWebServiceAddress(), optResult.get(), topic);
+                            return CompletableFuture.completedFuture(optResult);
+                        }
+                        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+                            return loadManager.get().findBrokerServiceUrl(Optional.of(topic), bundle);
+                        } else {
+                            // TODO: Add unit tests cover it.
+                            return findBrokerServiceUrl(bundle, options);
+                        }
+                    });
                 });
 
         future.thenAccept(optResult -> {
@@ -271,22 +282,40 @@ private CompletableFuture> internalGetWebServiceUrl(Optional {
-            if (lookupResult.isPresent()) {
+        return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> {
+            if (optResult.isPresent()) {
+                LOG.info("[{}] Redirect lookup request to {} for topic {}",
+                        pulsar.getSafeWebServiceAddress(), optResult.get(), topic);
                 try {
-                    LookupData lookupData = lookupResult.get().getLookupData();
+                    LookupData lookupData = optResult.get().getLookupData();
                     final String redirectUrl = options.isRequestHttps()
                             ? lookupData.getHttpUrlTls() : lookupData.getHttpUrl();
-                    return Optional.of(new URL(redirectUrl));
+                    return CompletableFuture.completedFuture(Optional.of(new URL(redirectUrl)));
                 } catch (Exception e) {
                     // just log the exception, nothing else to do
                     LOG.warn("internalGetWebServiceUrl [{}]", e.getMessage(), e);
                 }
+                return CompletableFuture.completedFuture(Optional.empty());
             }
-            return Optional.empty();
+            CompletableFuture> future =
+                    ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)
+                    ? loadManager.get().findBrokerServiceUrl(topic, bundle) :
+                    findBrokerServiceUrl(bundle, options);
+
+            return future.thenApply(lookupResult -> {
+                if (lookupResult.isPresent()) {
+                    try {
+                        LookupData lookupData = lookupResult.get().getLookupData();
+                        final String redirectUrl = options.isRequestHttps()
+                                ? lookupData.getHttpUrlTls() : lookupData.getHttpUrl();
+                        return Optional.of(new URL(redirectUrl));
+                    } catch (Exception e) {
+                        // just log the exception, nothing else to do
+                        LOG.warn("internalGetWebServiceUrl [{}]", e.getMessage(), e);
+                    }
+                }
+                return Optional.empty();
+            });
         });
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
index 05a156f87de11..26986a494f0be 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions;
 
+import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
@@ -379,19 +380,19 @@ public void testCloseRegister() throws Exception {
     public void testIsVerifiedNotification() {
         assertFalse(BrokerRegistryImpl.isVerifiedNotification(new Notification(NotificationType.Created, "/")));
         assertFalse(BrokerRegistryImpl.isVerifiedNotification(new Notification(NotificationType.Created,
-                BrokerRegistryImpl.LOOKUP_DATA_PATH + "xyz")));
+                LOADBALANCE_BROKERS_ROOT + "xyz")));
         assertFalse(BrokerRegistryImpl.isVerifiedNotification(new Notification(NotificationType.Created,
-                BrokerRegistryImpl.LOOKUP_DATA_PATH)));
+                LOADBALANCE_BROKERS_ROOT)));
         assertTrue(BrokerRegistryImpl.isVerifiedNotification(
-                new Notification(NotificationType.Created, BrokerRegistryImpl.LOOKUP_DATA_PATH + "/brokerId")));
+                new Notification(NotificationType.Created, LOADBALANCE_BROKERS_ROOT + "/brokerId")));
         assertTrue(BrokerRegistryImpl.isVerifiedNotification(
-                new Notification(NotificationType.Created, BrokerRegistryImpl.LOOKUP_DATA_PATH + "/brokerId/xyz")));
+                new Notification(NotificationType.Created, LOADBALANCE_BROKERS_ROOT + "/brokerId/xyz")));
     }
 
     @Test
     public void testKeyPath() {
         String keyPath = BrokerRegistryImpl.keyPath("brokerId");
-        assertEquals(keyPath, BrokerRegistryImpl.LOOKUP_DATA_PATH + "/brokerId");
+        assertEquals(keyPath, LOADBALANCE_BROKERS_ROOT + "/brokerId");
     }
 
     public BrokerRegistryImpl.State getState(BrokerRegistryImpl brokerRegistry) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index ae7ceeed92838..9ab2467d0ef1b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -473,7 +473,8 @@ public Map filter(Map broker
     }
 
     @Test
-    public void testStartOldLoadManager() throws Exception {
+    public void testDeployAndRollbackLoadManager() throws Exception {
+        // Test rollback to modular load manager.
         ServiceConfiguration defaultConf = getDefaultConf();
         defaultConf.setAllowAutoTopicCreation(true);
         defaultConf.setForceDeleteNamespaceAllowed(true);
@@ -482,20 +483,86 @@ public void testStartOldLoadManager() throws Exception {
         try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) {
             // start pulsar3 with old load manager
             var pulsar3 = additionalPulsarTestContext.getPulsarService();
+            String topic = "persistent://public/default/test";
+
+            String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic);
+            assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl());
+
+            String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic);
+            String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic);
+            assertEquals(lookupResult1, lookupResult2);
+            assertEquals(lookupResult1, lookupResult3);
+
+            NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test")).get();
+            LookupOptions options = LookupOptions.builder()
+                    .authoritative(false)
+                    .requestHttps(false)
+                    .readOnly(false)
+                    .loadTopicsInBundle(false).build();
+            Optional webServiceUrl1 =
+                    pulsar1.getNamespaceService().getWebServiceUrl(bundle, options);
+            assertTrue(webServiceUrl1.isPresent());
+            assertEquals(webServiceUrl1.get().toString(), pulsar3.getWebServiceAddress());
+
+            Optional webServiceUrl2 =
+                    pulsar2.getNamespaceService().getWebServiceUrl(bundle, options);
+            assertTrue(webServiceUrl2.isPresent());
+            assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString());
+
+            Optional webServiceUrl3 =
+                    pulsar3.getNamespaceService().getWebServiceUrl(bundle, options);
+            assertTrue(webServiceUrl3.isPresent());
+            assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString());
+
+            // Test deploy new broker with new load manager
+            ServiceConfiguration conf = getDefaultConf();
+            conf.setAllowAutoTopicCreation(true);
+            conf.setForceDeleteNamespaceAllowed(true);
+            conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+            conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
+            try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) {
+                var pulsar4 = additionPulsarTestContext.getPulsarService();
+
+                Set availableCandidates = Sets.newHashSet(pulsar1.getBrokerServiceUrl(),
+                        pulsar2.getBrokerServiceUrl(),
+                        pulsar4.getBrokerServiceUrl());
+                String lookupResult4 = pulsar4.getAdminClient().lookups().lookupTopic(topic);
+                assertTrue(availableCandidates.contains(lookupResult4));
+
+                String lookupResult5 = pulsar1.getAdminClient().lookups().lookupTopic(topic);
+                String lookupResult6 = pulsar2.getAdminClient().lookups().lookupTopic(topic);
+                String lookupResult7 = pulsar3.getAdminClient().lookups().lookupTopic(topic);
+                assertEquals(lookupResult4, lookupResult5);
+                assertEquals(lookupResult4, lookupResult6);
+                assertEquals(lookupResult4, lookupResult7);
+
+                Set availableWebUrlCandidates = Sets.newHashSet(pulsar1.getWebServiceAddress(),
+                        pulsar2.getWebServiceAddress(),
+                        pulsar4.getWebServiceAddress());
+
+                webServiceUrl1 =
+                        pulsar1.getNamespaceService().getWebServiceUrl(bundle, options);
+                assertTrue(webServiceUrl1.isPresent());
+                assertTrue(availableWebUrlCandidates.contains(webServiceUrl1.get().toString()));
+
+                webServiceUrl2 =
+                        pulsar2.getNamespaceService().getWebServiceUrl(bundle, options);
+                assertTrue(webServiceUrl2.isPresent());
+                assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString());
+
+                // The pulsar3 will redirect to pulsar4
+                webServiceUrl3 =
+                        pulsar3.getNamespaceService().getWebServiceUrl(bundle, options);
+                assertTrue(webServiceUrl3.isPresent());
+                // It will redirect to pulsar4
+                assertTrue(availableWebUrlCandidates.contains(webServiceUrl3.get().toString()));
+
+                var webServiceUrl4 =
+                        pulsar4.getNamespaceService().getWebServiceUrl(bundle, options);
+                assertTrue(webServiceUrl4.isPresent());
+                assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString());
 
-            var availableBrokers = pulsar3.getLoadManager().get().getAvailableBrokers();
-            assertEquals(availableBrokers.size(), 1);
-            assertEquals(availableBrokers.iterator().next(), pulsar3.getLookupServiceAddress());
-
-            availableBrokers = pulsar1.getLoadManager().get().getAvailableBrokers();
-            assertEquals(availableBrokers.size(), 2);
-            assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress()));
-            assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress()));
-
-            availableBrokers = pulsar2.getLoadManager().get().getAvailableBrokers();
-            assertEquals(availableBrokers.size(), 2);
-            assertTrue(availableBrokers.contains(pulsar1.getLookupServiceAddress()));
-            assertTrue(availableBrokers.contains(pulsar2.getLookupServiceAddress()));
+            }
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java
index b5d2d63a31156..7bcd0687f0008 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.data;
 
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
 import org.junit.Assert;
@@ -41,7 +42,8 @@ public void testConstructors() {
         }};
         BrokerLookupData lookupData = new BrokerLookupData(
                 webServiceUrl, webServiceUrlTls, pulsarServiceUrl,
-                pulsarServiceUrlTls, advertisedListeners, protocols, true, true, "3.0");
+                pulsarServiceUrlTls, advertisedListeners, protocols, true, true,
+                ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(),"3.0");
         Assert.assertEquals(webServiceUrl, lookupData.webServiceUrl());
         Assert.assertEquals(webServiceUrlTls, lookupData.webServiceUrlTls());
         Assert.assertEquals(pulsarServiceUrl, lookupData.pulsarServiceUrl());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
index c521861471c9a..68bd7b29094cd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
@@ -30,6 +30,7 @@
 import java.util.function.BiConsumer;
 
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
@@ -105,6 +106,10 @@ public BrokerLookupData getLookupData() {
     }
 
     public BrokerLookupData getLookupData(String version) {
+        return getLookupData(version, ExtensibleLoadManagerImpl.class.getName());
+    }
+
+    public BrokerLookupData getLookupData(String version, String loadManagerClassName) {
         String webServiceUrl = "http://localhost:8080";
         String webServiceUrlTls = "https://localhoss:8081";
         String pulsarServiceUrl = "pulsar://localhost:6650";
@@ -115,6 +120,7 @@ public BrokerLookupData getLookupData(String version) {
         }};
         return new BrokerLookupData(
                 webServiceUrl, webServiceUrlTls, pulsarServiceUrl,
-                pulsarServiceUrlTls, advertisedListeners, protocols, true, true, version);
+                pulsarServiceUrlTls, advertisedListeners, protocols, true, true,
+                loadManagerClassName, -1, version);
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java
index 5bc1e436bae25..c2c534f72e9db 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java
@@ -33,6 +33,7 @@
 import java.util.Set;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
@@ -211,7 +212,8 @@ public BrokerLookupData getLookupData(boolean persistentTopicsEnabled,
         return new BrokerLookupData(
                 webServiceUrl, webServiceUrlTls, pulsarServiceUrl,
                 pulsarServiceUrlTls, advertisedListeners, protocols,
-                persistentTopicsEnabled, nonPersistentTopicsEnabled, "3.0.0");
+                persistentTopicsEnabled, nonPersistentTopicsEnabled,
+                ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(), "3.0.0");
     }
 
     public LoadManagerContext getContext() {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
new file mode 100644
index 0000000000000..0169b57fe993e
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.filter;
+
+import static org.testng.Assert.assertEquals;
+import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
+import org.testng.annotations.Test;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * Unit test for {@link BrokerLoadManagerClassFilter}.
+ */
+public class BrokerLoadManagerClassFilterTest extends BrokerFilterTestBase {
+
+    @Test
+    public void test() throws BrokerFilterException {
+        LoadManagerContext context = getContext();
+        context.brokerConfiguration().setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+        BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter();
+
+        Map originalBrokers = Map.of(
+                "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
+                "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
+                "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()),
+                "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName())
+        );
+
+        Map result = filter.filter(new HashMap<>(originalBrokers), null, context);
+        assertEquals(result, Map.of(
+                "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
+                "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName())
+        ));
+
+        context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+        result = filter.filter(new HashMap<>(originalBrokers), null, context);
+
+        assertEquals(result, Map.of(
+                "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()),
+                "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName())
+        ));
+
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index d673722d6fb44..af5890fcacbb2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -664,7 +664,8 @@ public BrokerLookupData getLookupData() {
         return new BrokerLookupData(
                 webServiceUrl, webServiceUrlTls, pulsarServiceUrl,
                 pulsarServiceUrlTls, advertisedListeners, protocols,
-                true, true, "3.0.0");
+                true, true,
+                conf.getLoadManagerClassName(), System.currentTimeMillis(), "3.0.0");
     }
 
     private void setIsolationPolicies(SimpleResourceAllocationPolicies policies,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
new file mode 100644
index 0000000000000..856bbac029226
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.impl;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.LoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.policies.data.loadbalancer.BrokerData;
+import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData;
+import org.testng.annotations.Test;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Unit test for {@link BrokerLoadManagerClassFilter}.
+ */
+public class BrokerLoadManagerClassFilterTest {
+
+    @Test
+    public void test() throws BrokerFilterException {
+        BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter();
+
+        LoadData loadData = new LoadData();
+        LocalBrokerData localBrokerData = new LocalBrokerData();
+        localBrokerData.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+
+        LocalBrokerData localBrokerData1 = new LocalBrokerData();
+        localBrokerData1.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+        loadData.getBrokerData().put("broker1", new BrokerData(localBrokerData));
+        loadData.getBrokerData().put("broker2", new BrokerData(localBrokerData1));
+
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+
+        Set brokers = new HashSet<>(){{
+            add("broker1");
+            add("broker2");
+        }};
+        filter.filter(brokers, null, loadData, conf);
+
+        assertEquals(brokers.size(), 1);
+        assertTrue(brokers.contains("broker1"));
+
+        brokers = new HashSet<>(){{
+            add("broker1");
+            add("broker2");
+        }};
+        conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+        filter.filter(brokers, null, loadData, conf);
+        assertEquals(brokers.size(), 1);
+        assertTrue(brokers.contains("broker2"));
+    }
+}
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java
index 2d138edd924bc..bf4a11c8f875d 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ServiceLookupData.java
@@ -48,4 +48,8 @@ public interface ServiceLookupData {
      */
     Optional getProtocol(String protocol);
 
+    String getLoadManagerClassName();
+
+    long getStartTimestamp();
+
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java
index 75fddc21bf91b..6e519a3f0735f 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java
@@ -55,6 +55,8 @@ public class LoadReport implements LoadManagerReport {
     private int numProducers;
     private int numBundles;
     private Map protocols;
+    private String loadManagerClassName;
+    private long startTimestamp;
     // This place-holder requires to identify correct LoadManagerReport type while deserializing
     @SuppressWarnings("checkstyle:ConstantName")
     public static final String loadReportType = LoadReport.class.getSimpleName();
@@ -474,4 +476,22 @@ public void setProtocols(Map protocols) {
     public Optional getProtocol(String protocol) {
         return Optional.ofNullable(protocols.get(protocol));
     }
+
+    @Override
+    public String getLoadManagerClassName() {
+        return this.loadManagerClassName;
+    }
+
+    public void setLoadManagerClassName(String loadManagerClassName) {
+        this.loadManagerClassName = loadManagerClassName;
+    }
+
+    @Override
+    public long getStartTimestamp() {
+        return this.startTimestamp;
+    }
+
+    public void setStartTimestamp(long startTimestamp) {
+        this.startTimestamp = startTimestamp;
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java
index 8c0c008e0a555..df85a4d989f99 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java
@@ -91,6 +91,9 @@ public class LocalBrokerData implements LoadManagerReport {
     //
     private Map advertisedListeners;
 
+    private String loadManagerClassName;
+    private long startTimestamp;
+
     // For JSON only.
     public LocalBrokerData() {
         this(null, null, null, null);
@@ -113,6 +116,7 @@ public LocalBrokerData(final String webServiceUrl, final String webServiceUrlTls
         this.pulsarServiceUrlTls = pulsarServiceUrlTls;
         lastStats = new ConcurrentHashMap<>();
         lastUpdate = System.currentTimeMillis();
+        startTimestamp = System.currentTimeMillis();
         cpu = new ResourceUsage();
         memory = new ResourceUsage();
         directMemory = new ResourceUsage();
@@ -529,4 +533,16 @@ public Map getAdvertisedListeners() {
     public void setAdvertisedListeners(Map advertisedListeners) {
         this.advertisedListeners = advertisedListeners;
     }
+
+    public String getLoadManagerClassName() {
+        return this.loadManagerClassName;
+    }
+
+    public void setLoadManagerClassName(String loadManagerClassName) {
+        this.loadManagerClassName = loadManagerClassName;
+    }
+
+    public long getStartTimestamp() {
+        return this.startTimestamp;
+    }
 }

From 18ccd9ef22bb44d952875460fbacad45c8d8d26a Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com>
Date: Tue, 11 Apr 2023 19:28:25 -0700
Subject: [PATCH 180/404] [fix] Upgrade Vertx to match BK's version (otherwise
 BK does not work) (#20070)

### Motivation

BK deployed with Pulsar fail on start with
```
ERROR org.apache.bookkeeper.common.component.AbstractLifecycleComponent - Failed to start Component: http-service
java.lang.NoSuchMethodError: 'io.vertx.core.Future io.vertx.core.Vertx.deployVerticle(io.vertx.core.Verticle)'
	at org.apache.bookkeeper.http.vertx.VertxHttpServer.startServer(VertxHttpServer.java:93) ~[org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar:4.16.0]
	at org.apache.bookkeeper.server.service.HttpService.doStart(HttpService.java:62) ~[org.apache.bookkeeper-bookkeeper-server-4.16.0.jar:4.16.0]
	at org.apache.bookkeeper.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:83) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0]
	at org.apache.bookkeeper.common.component.LifecycleComponentStack.lambda$start$4(LifecycleComponentStack.java:144) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0]
	at com.google.common.collect.ImmutableList.forEach(ImmutableList.java:422) ~[com.google.guava-guava-31.0.1-jre.jar:?]
	at org.apache.bookkeeper.common.component.LifecycleComponentStack.start(LifecycleComponentStack.java:144) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0]
	at org.apache.bookkeeper.common.component.ComponentStarter.startComponent(ComponentStarter.java:84) ~[org.apache.bookkeeper-bookkeeper-common-4.16.0.jar:4.16.0]
	at org.apache.bookkeeper.server.Main.doMain(Main.java:224) ~[org.apache.bookkeeper-bookkeeper-server-4.16.0.jar:4.16.0]
	at org.apache.bookkeeper.server.Main.main(Main.java:199) ~[org.apache.bookkeeper-bookkeeper-server-4.16.0.jar:4.16.0]

```

The root cause is that BK upgraded to 4.16.0 and needs newer version of vertx that introduced API changes https://github.com/apache/bookkeeper/pull/3435

Pulsar forces older version of dependency.

### Modifications

Upgrade Vertx to the same version as BK
---
 distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++-----
 pom.xml                                          |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 4cb1d1544a63e..0d5cd7c02a23a 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -470,11 +470,11 @@ The Apache Software License, Version 2.0
   * JCTools - Java Concurrency Tools for the JVM
     - org.jctools-jctools-core-2.1.2.jar
   * Vertx
-    - io.vertx-vertx-auth-common-3.9.8.jar
-    - io.vertx-vertx-bridge-common-3.9.8.jar
-    - io.vertx-vertx-core-3.9.8.jar
-    - io.vertx-vertx-web-3.9.8.jar
-    - io.vertx-vertx-web-common-3.9.8.jar
+    - io.vertx-vertx-auth-common-4.3.8.jar
+    - io.vertx-vertx-bridge-common-4.3.8.jar
+    - io.vertx-vertx-core-4.3.8.jar
+    - io.vertx-vertx-web-4.3.8.jar
+    - io.vertx-vertx-web-common-4.3.8.jar
   * Apache ZooKeeper
     - org.apache.zookeeper-zookeeper-3.8.1.jar
     - org.apache.zookeeper-zookeeper-jute-3.8.1.jar
diff --git a/pom.xml b/pom.xml
index 389d9f6f83487..404807bc4ad7c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@ flexible messaging model and an intuitive client API.
     2.34
     1.10.50
     0.16.0
-    3.9.8
+    4.3.8
     6.29.4.1
     1.7.32
     4.4

From cff3f9baa03f166697d31ff517f234ecf0fc702d Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Wed, 12 Apr 2023 11:00:14 +0800
Subject: [PATCH 181/404] [fix][client] Cache empty schema version in
 ProducerImpl schemaCache. (#19929)

Co-authored-by: wangjinlong 
---
 .../pulsar/client/api/MockBrokerService.java  |  21 ++++
 .../client/api/MockBrokerServiceHooks.java    |   5 +
 .../impl/ProducerEmptySchemaCacheTest.java    | 100 ++++++++++++++++++
 .../pulsar/client/impl/ProducerImpl.java      |  23 +++-
 4 files changed, 144 insertions(+), 5 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java
index 3e6ee4f2a969d..9ca0bafe00b3b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java
@@ -44,6 +44,7 @@
 import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandCloseProducerHook;
 import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandConnectHook;
 import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandFlowHook;
+import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandGetOrCreateSchemaHook;
 import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandPartitionLookupHook;
 import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandProducerHook;
 import org.apache.pulsar.client.api.MockBrokerServiceHooks.CommandSendHook;
@@ -55,6 +56,7 @@
 import org.apache.pulsar.common.api.proto.CommandCloseProducer;
 import org.apache.pulsar.common.api.proto.CommandConnect;
 import org.apache.pulsar.common.api.proto.CommandFlow;
+import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema;
 import org.apache.pulsar.common.api.proto.CommandLookupTopic;
 import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType;
 import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata;
@@ -77,6 +79,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
+ *
  */
 public class MockBrokerService {
     private LookupData lookupData;
@@ -244,6 +247,19 @@ protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) {
             ctx.writeAndFlush(Commands.newSuccess(closeConsumer.getRequestId()));
         }
 
+        @Override
+        protected void handleGetOrCreateSchema(CommandGetOrCreateSchema commandGetOrCreateSchema) {
+            if (handleGetOrCreateSchema != null) {
+                handleGetOrCreateSchema.apply(ctx, commandGetOrCreateSchema);
+                return;
+            }
+
+            // default
+            ctx.writeAndFlush(
+                    Commands.newGetOrCreateSchemaResponse(commandGetOrCreateSchema.getRequestId(),
+                            SchemaVersion.Empty));
+        }
+
         @Override
         public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
             log.warn("Got exception", cause);
@@ -276,6 +292,7 @@ final protected void handlePong(CommandPong pong) {
     private CommandUnsubscribeHook handleUnsubscribe = null;
     private CommandCloseProducerHook handleCloseProducer = null;
     private CommandCloseConsumerHook handleCloseConsumer = null;
+    private CommandGetOrCreateSchemaHook handleGetOrCreateSchema = null;
 
     public MockBrokerService() {
         server = new Server(0);
@@ -416,6 +433,10 @@ public void setHandleCloseConsumer(CommandCloseConsumerHook hook) {
         handleCloseConsumer = hook;
     }
 
+    public void setHandleGetOrCreateSchema(CommandGetOrCreateSchemaHook hook) {
+        handleGetOrCreateSchema = hook;
+    }
+
     public void resetHandleCloseConsumer() {
         handleCloseConsumer = null;
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java
index 06289e123c8b9..dd2058ce5a485 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerServiceHooks.java
@@ -26,6 +26,7 @@
 import org.apache.pulsar.common.api.proto.CommandCloseProducer;
 import org.apache.pulsar.common.api.proto.CommandConnect;
 import org.apache.pulsar.common.api.proto.CommandFlow;
+import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema;
 import org.apache.pulsar.common.api.proto.CommandLookupTopic;
 import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata;
 import org.apache.pulsar.common.api.proto.CommandProducer;
@@ -77,4 +78,8 @@ interface CommandCloseProducerHook {
     interface CommandCloseConsumerHook {
         void apply(ChannelHandlerContext ctx, CommandCloseConsumer closeConsumer);
     }
+
+    interface CommandGetOrCreateSchemaHook {
+        void apply(ChannelHandlerContext ctx, CommandGetOrCreateSchema closeConsumer);
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java
new file mode 100644
index 0000000000000..079b8bd8eccbd
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerEmptySchemaCacheTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl;
+
+import lombok.Cleanup;
+import org.apache.pulsar.client.api.MockBrokerService;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.TypedMessageBuilder;
+import org.apache.pulsar.common.protocol.Commands;
+import org.apache.pulsar.common.protocol.schema.SchemaVersion;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.testng.Assert.assertEquals;
+
+@Test(groups = "broker-impl")
+public class ProducerEmptySchemaCacheTest {
+
+    MockBrokerService mockBrokerService;
+
+    @BeforeClass(alwaysRun = true)
+    public void setup() {
+        mockBrokerService = new MockBrokerService();
+        mockBrokerService.start();
+    }
+
+    @AfterClass(alwaysRun = true)
+    public void teardown() {
+        mockBrokerService.stop();
+    }
+
+    @Test
+    public void testProducerShouldCacheEmptySchema() throws Exception {
+        @Cleanup
+        PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder()
+                .serviceUrl(mockBrokerService.getBrokerAddress())
+                .build();
+
+        AtomicLong counter = new AtomicLong(0);
+
+        mockBrokerService.setHandleGetOrCreateSchema((ctx, commandGetOrCreateSchema) -> {
+            counter.incrementAndGet();
+            ctx.writeAndFlush(
+                    Commands.newGetOrCreateSchemaResponse(commandGetOrCreateSchema.getRequestId(),
+                            SchemaVersion.Empty));
+        });
+
+        // this schema mode is used in consumer retry and dlq Producer
+        // when the origin consumer has Schema.BYTES schema
+        // and when retry message or dlq message is send
+        // will use typed message builder set Schema.Bytes to send message.
+
+        Schema schema = Schema.BYTES;
+        Schema readerSchema = Schema.BYTES;
+
+        @Cleanup
+        Producer dlqProducer = client.newProducer(Schema.AUTO_PRODUCE_BYTES(schema))
+                .topic("testAutoProduceBytesSchemaShouldCache")
+                .sendTimeout(5, TimeUnit.SECONDS)
+                .maxPendingMessages(0)
+                .enableBatching(false)
+                .create();
+
+        for (int i = 10; i > 0; i--) {
+            TypedMessageBuilder typedMessageBuilderNew =
+                    dlqProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(readerSchema))
+                            .value("hello".getBytes());
+
+            typedMessageBuilderNew.send();
+        }
+
+        // schema should only be requested once.
+        // and if the schemaVersion is empty (e.g. Schema.BYTES)
+        // it should be cached by the client
+        // to avoid continuously send `CommandGetOrCreateSchema` rpc
+        assertEquals(counter.get(), 1);
+    }
+}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
index b8f249dbc0e4b..2192ebfb64e75 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
@@ -88,6 +88,7 @@
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.Commands.ChecksumType;
 import org.apache.pulsar.common.protocol.schema.SchemaHash;
+import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.util.DateFormatter;
@@ -733,11 +734,16 @@ boolean populateMessageSchema(MessageImpl msg, SendCallback callback) {
             completeCallbackAndReleaseSemaphore(msg.getUncompressedSize(), callback, e);
             return false;
         }
+
         byte[] schemaVersion = schemaCache.get(msg.getSchemaHash());
         if (schemaVersion != null) {
-            msgMetadataBuilder.setSchemaVersion(schemaVersion);
+            if (schemaVersion != SchemaVersion.Empty.bytes()) {
+                msgMetadataBuilder.setSchemaVersion(schemaVersion);
+            }
+
             msg.setSchemaState(MessageImpl.SchemaState.Ready);
         }
+
         return true;
     }
 
@@ -746,7 +752,11 @@ private boolean rePopulateMessageSchema(MessageImpl msg) {
         if (schemaVersion == null) {
             return false;
         }
-        msg.getMessageBuilder().setSchemaVersion(schemaVersion);
+
+        if (schemaVersion != SchemaVersion.Empty.bytes()) {
+            msg.getMessageBuilder().setSchemaVersion(schemaVersion);
+        }
+
         msg.setSchemaState(MessageImpl.SchemaState.Ready);
         return true;
     }
@@ -769,12 +779,15 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call
                 }
             } else {
                 log.info("[{}] [{}] GetOrCreateSchema succeed", topic, producerName);
-                // In broker, if schema version is an empty byte array, it means the topic doesn't have schema. In this
-                // case, we should not cache the schema version so that the schema version of the message metadata will
-                // be null, instead of an empty array.
+                // In broker, if schema version is an empty byte array, it means the topic doesn't have schema.
+                // In this case, we cache the schema version to `SchemaVersion.Empty.bytes()`.
+                // When we need to set the schema version of the message metadata,
+                // we should check if the cached schema version is `SchemaVersion.Empty.bytes()`
                 if (v.length != 0) {
                     schemaCache.putIfAbsent(msg.getSchemaHash(), v);
                     msg.getMessageBuilder().setSchemaVersion(v);
+                } else {
+                    schemaCache.putIfAbsent(msg.getSchemaHash(), SchemaVersion.Empty.bytes());
                 }
                 msg.setSchemaState(MessageImpl.SchemaState.Ready);
             }

From 1e9c7f5236866d07105421c566241e3798f63097 Mon Sep 17 00:00:00 2001
From: Apurva007 
Date: Tue, 11 Apr 2023 20:07:45 -0700
Subject: [PATCH 182/404] [Fix][Doc] Update docs for Partition increase command
 (#20035)

Co-authored-by: Apurva Telang 
---
 .../pulsar/broker/admin/v1/PersistentTopics.java     |  4 ++--
 .../pulsar/broker/admin/v2/PersistentTopics.java     |  4 ++--
 .../java/org/apache/pulsar/client/admin/Topics.java  | 12 ++++++------
 .../apache/pulsar/admin/cli/CmdPersistentTopics.java |  2 +-
 .../java/org/apache/pulsar/admin/cli/CmdTopics.java  |  2 +-
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java
index 16f1b357dcd38..e9bb1a4054764 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java
@@ -256,14 +256,14 @@ public void createNonPartitionedTopic(
     }
 
     /**
-     * It updates number of partitions of an existing non-global partitioned topic. It requires partitioned-topic to be
+     * It updates number of partitions of an existing partitioned topic. It requires partitioned-topic to be
      * already exist and number of new partitions must be greater than existing number of partitions. Decrementing
      * number of partitions requires deletion of topic which is not supported.
      */
     @POST
     @Path("/{property}/{cluster}/{namespace}/{topic}/partitions")
     @ApiOperation(hidden = true, value = "Increment partitions of an existing partitioned topic.",
-            notes = "It only increments partitions of existing non-global partitioned-topic")
+            notes = "It increments partitions of existing partitioned-topic")
     @ApiResponses(value = {
             @ApiResponse(code = 204, message = "Update topic partition successful."),
             @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index 477cc8b57129d..eee363aeed86b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -775,14 +775,14 @@ public void deleteDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRe
     }
 
     /**
-     * It updates number of partitions of an existing non-global partitioned topic. It requires partitioned-topic to be
+     * It updates number of partitions of an existing partitioned topic. It requires partitioned-topic to be
      * already exist and number of new partitions must be greater than existing number of partitions. Decrementing
      * number of partitions requires deletion of topic which is not supported.
      */
     @POST
     @Path("/{tenant}/{namespace}/{topic}/partitions")
     @ApiOperation(value = "Increment partitions of an existing partitioned topic.",
-            notes = "It only increments partitions of existing non-global partitioned-topic")
+            notes = "It increments partitions of existing partitioned-topic")
     @ApiResponses(value = {
             @ApiResponse(code = 204, message = "Update topic partition successful."),
             @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java
index 0d5d14eb73463..f599e2566bffc 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java
@@ -589,7 +589,7 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) {
     CompletableFuture createMissedPartitionsAsync(String topic);
 
     /**
-     * Update number of partitions of a non-global partitioned topic.
+     * Update number of partitions of a partitioned topic.
      * 

      * It requires partitioned-topic to be already exist and number of new partitions must be greater than existing
      * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported.
@@ -605,7 +605,7 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) {
     void updatePartitionedTopic(String topic, int numPartitions) throws PulsarAdminException;
 
     /**
-     * Update number of partitions of a non-global partitioned topic asynchronously.
+     * Update number of partitions of a partitioned topic asynchronously.
      * 

      * It requires partitioned-topic to be already exist and number of new partitions must be greater than existing
      * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported.
@@ -621,7 +621,7 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) {
     CompletableFuture updatePartitionedTopicAsync(String topic, int numPartitions);
 
     /**
-     * Update number of partitions of a non-global partitioned topic.
+     * Update number of partitions of a partitioned topic.
      * 

      * It requires partitioned-topic to be already exist and number of new partitions must be greater than existing
      * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported.
@@ -641,7 +641,7 @@ void updatePartitionedTopic(String topic, int numPartitions, boolean updateLocal
             throws PulsarAdminException;
 
     /**
-     * Update number of partitions of a non-global partitioned topic asynchronously.
+     * Update number of partitions of a partitioned topic asynchronously.
      * 

      * It requires partitioned-topic to be already exist and number of new partitions must be greater than existing
      * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported.
@@ -661,7 +661,7 @@ CompletableFuture updatePartitionedTopicAsync(String topic, int numPartiti
             boolean force);
 
     /**
-     * Update number of partitions of a non-global partitioned topic.
+     * Update number of partitions of a partitioned topic.
      * 

      * It requires partitioned-topic to be already exist and number of new partitions must be greater than existing
      * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported.
@@ -679,7 +679,7 @@ void updatePartitionedTopic(String topic, int numPartitions, boolean updateLocal
             throws PulsarAdminException;
 
     /**
-     * Update number of partitions of a non-global partitioned topic asynchronously.
+     * Update number of partitions of a partitioned topic asynchronously.
      * 

      * It requires partitioned-topic to be already exist and number of new partitions must be greater than existing
      * number of partitions. Decrementing number of partitions requires deletion of topic which is not supported.
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java
index 6e59be39c018b..3c1662d00a034 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java
@@ -211,7 +211,7 @@ void run() throws Exception {
         }
     }
 
-    @Parameters(commandDescription = "Update existing non-global partitioned topic. "
+    @Parameters(commandDescription = "Update existing partitioned topic. "
             + "New updating number of partitions must be greater than existing number of partitions.")
     private class UpdatePartitionedCmd extends CliCommand {
 
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java
index 8b11990dc531b..f96410749aea6 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java
@@ -585,7 +585,7 @@ void run() throws Exception {
         }
     }
 
-    @Parameters(commandDescription = "Update existing non-global partitioned topic. "
+    @Parameters(commandDescription = "Update existing partitioned topic. "
             + "New updating number of partitions must be greater than existing number of partitions.")
     private class UpdatePartitionedCmd extends CliCommand {
 

From a332fead21390b3fbbb86c2d17a161717a170161 Mon Sep 17 00:00:00 2001
From: Hang Chen 
Date: Wed, 12 Apr 2023 13:12:46 +0800
Subject: [PATCH 183/404] Upgrade the RocksDB version to 7.9.2 to keep sync
 with BookKeeper's RocksDB dependency (#20072)

### Motivation
BookKeeper has upgraded the RocksDB dependency to 7.9.2, related discussion:
https://lists.apache.org/thread/8j90y4vrvgz1nvt5pb0xdjjy3o8z57z7

https://github.com/apache/bookkeeper/issues/3734

However, Pulsar also has the RocksDB dependency and it will override the BookKeeper's RocksDB dependency version. It will lead to the release package still using the old RocksDB version (6.29.4.1)

### Modifications
Upgrade the Pulsar's RocksDB dependency to 7.9.2 to keep sync with the BookKeeper's RocksDB dependency.
---
 distribution/server/src/assemble/LICENSE.bin.txt | 2 +-
 pom.xml                                          | 2 +-
 pulsar-sql/presto-distribution/LICENSE           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 0d5cd7c02a23a..c321e54de2e01 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -403,7 +403,7 @@ The Apache Software License, Version 2.0
     - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar
     - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar
  * SnakeYaml -- org.yaml-snakeyaml-1.32.jar
- * RocksDB - org.rocksdb-rocksdbjni-6.29.4.1.jar
+ * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
  * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar
  * OkHttp3
diff --git a/pom.xml b/pom.xml
index 404807bc4ad7c..094a35b5ed13a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -141,7 +141,7 @@ flexible messaging model and an intuitive client API.
     1.10.50
     0.16.0
     4.3.8
-    6.29.4.1
+    7.9.2
     1.7.32
     4.4
     2.18.0
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index e101191bbf536..214a928a74941 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -399,7 +399,7 @@ The Apache Software License, Version 2.0
     - trino-spi-368.jar
     - trino-record-decoder-368.jar
   * RocksDB JNI
-    - rocksdbjni-6.29.4.1.jar
+    - rocksdbjni-7.9.2.jar
   * SnakeYAML
     - snakeyaml-1.32.jar
   * Bean Validation API

From 3efdc9af3bede77b018146caf1cd11730701ba4f Mon Sep 17 00:00:00 2001
From: Ayman Khalil <13960949+aymkhalil@users.noreply.github.com>
Date: Wed, 12 Apr 2023 06:09:23 -0700
Subject: [PATCH 184/404] [improve][pulsar-shell] Use singleton custom command
 factories (#20064)

---
 .../pulsar/admin/cli/PulsarAdminToolTest.java | 31 +++++++++++++++++++
 .../pulsar/admin/cli/PulsarAdminTool.java     | 10 ++----
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
index ccae1b1176527..2fe7b0236f0e8 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
@@ -29,8 +29,11 @@
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertSame;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
+import static org.testng.AssertJUnit.assertNotNull;
+
 import com.beust.jcommander.JCommander;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
@@ -45,12 +48,14 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory;
 import org.apache.pulsar.admin.cli.utils.SchemaExtractor;
 import org.apache.pulsar.client.admin.Bookies;
 import org.apache.pulsar.client.admin.BrokerStats;
@@ -2438,6 +2443,32 @@ public void customCommands() throws Exception {
 
     }
 
+    @Test
+    public void customCommandsFactoryImmutable() throws Exception {
+        File narFile = new File(PulsarAdminTool.class.getClassLoader()
+                .getResource("cliextensions/customCommands-nar.nar").getFile());
+        log.info("NAR FILE is {}", narFile);
+
+        PulsarAdminBuilder builder = mock(PulsarAdminBuilder.class);
+        PulsarAdmin admin = mock(PulsarAdmin.class);
+        when(builder.build()).thenReturn(admin);
+        Topics topics = mock(Topics.class);
+        when(admin.topics()).thenReturn(topics);
+        TopicStats topicStats = mock(TopicStats.class);
+        when(topics.getStats(anyString())).thenReturn(topicStats);
+        when(topicStats.toString()).thenReturn("MOCK-TOPIC-STATS");
+
+        Properties properties = new Properties();
+        properties.put("webServiceUrl", "http://localhost:2181");
+        properties.put("cliExtensionsDirectory", narFile.getParentFile().getAbsolutePath());
+        properties.put("customCommandFactories", "dummy");
+        PulsarAdminTool tool = new PulsarAdminTool(properties);
+        List customCommandFactories = tool.customCommandFactories;
+        assertNotNull(customCommandFactories);
+        tool.run(split("-h"));
+        assertSame(tool.customCommandFactories, customCommandFactories);
+    }
+
     @Test
     public void testHelpFlag() {
         PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
index ca0a8a055cfc9..c06016be43883 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
@@ -25,7 +25,6 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.io.FileInputStream;
 import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -49,7 +48,7 @@ public class PulsarAdminTool {
 
     private static int lastExitCode = Integer.MIN_VALUE;
 
-    protected List customCommandFactories = new ArrayList();
+    protected final List customCommandFactories;
     protected Map> commandMap;
     protected JCommander jcommander;
     protected RootParams rootParams;
@@ -101,6 +100,7 @@ public static class RootParams {
 
     public PulsarAdminTool(Properties properties) throws Exception {
         this.properties = properties;
+        customCommandFactories = CustomCommandFactoryProvider.createCustomCommandFactories(properties);
         rootParams = new RootParams();
         // fallback to previous-version serviceUrl property to maintain backward-compatibility
         initRootParamsFromProperties(properties);
@@ -169,7 +169,6 @@ public Properties getConfiguration() {
                     return properties;
                 }
             };
-            loadCustomCommandFactories();
 
             for (CustomCommandFactory factory : customCommandFactories) {
                 List customCommandGroups = factory.commandGroups(context);
@@ -191,11 +190,6 @@ public Properties getConfiguration() {
         }
     }
 
-    private void loadCustomCommandFactories() throws Exception {
-        customCommandFactories = CustomCommandFactoryProvider.createCustomCommandFactories(properties);
-    }
-
-
     private void addCommand(Map.Entry> c, Supplier admin) throws Exception {
         // To remain backwards compatibility for "source" and "sink" commands
         // TODO eventually remove this

From 117fad54bc67e8c9342cd220080eaf52930414ac Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 12 Apr 2023 10:22:05 -0500
Subject: [PATCH 185/404] [improve] Protect branch-3.0 (#20080)

### Motivation

Add branch protection to the new `branch-3.0`.

### Modifications

* Updated the `.asf.yaml` to include `branch-3.0`.

### Documentation

- [x] `doc-not-needed`
---
 .asf.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.asf.yaml b/.asf.yaml
index bec521f1a8c71..459dcb6e2fe57 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -82,6 +82,7 @@ github:
     branch-2.9: {}
     branch-2.10: {}
     branch-2.11: {}
+    branch-3.0: {}
 
 notifications:
   commits:      commits@pulsar.apache.org

From 547d792439e7b8bfefb0236b078ed145731da658 Mon Sep 17 00:00:00 2001
From: ken <1647023764@qq.com>
Date: Thu, 13 Apr 2023 01:22:02 +0800
Subject: [PATCH 186/404] [fix] [broker] error TimeUnit to record publish
 latency (#20074)

Co-authored-by: fanjianye 
Co-authored-by: tison 
---
 .../apache/pulsar/broker/rest/RestMessagePublishContext.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java
index d7cee8b600b31..3c9adbd3e4fe4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java
@@ -53,7 +53,7 @@ public void completed(Exception exception, long ledgerId, long entryId) {
                                 + "triggered send callback.",
                         topic.getName(), ledgerId, entryId);
             }
-            topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.MICROSECONDS);
+            topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.NANOSECONDS);
             positionFuture.complete(PositionImpl.get(ledgerId, entryId));
         }
         recycle();

From 52e8144587548a692e550a8538f4d2667b5499d6 Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Wed, 12 Apr 2023 20:28:42 -0400
Subject: [PATCH 187/404] [fix][fn] check user metric len before iterating
 (#20021)

Co-authored-by: Andy Walker 
---
 pulsar-function-go/pf/instance.go             |  3 ++
 .../pf/instanceControlServicer_test.go        | 53 +++++++++++++++++++
 pulsar-function-go/pf/stats_test.go           | 27 ++++++++++
 3 files changed, 83 insertions(+)

diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 5d17cfe0c333a..a82273031ecfb 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -669,6 +669,9 @@ func (gi *goInstance) getTotalReceived1min() float32 {
 func (gi *goInstance) getUserMetricsMap() map[string]float64 {
 	userMetricMap := map[string]float64{}
 	filteredMetricFamilies := gi.getFilteredMetricFamilies(PulsarFunctionMetricsPrefix + UserMetric)
+	if len(filteredMetricFamilies) == 0 {
+		return userMetricMap
+	}
 	for _, m := range filteredMetricFamilies[0].GetMetric() {
 		var isFuncMetric bool
 		var userLabelName string
diff --git a/pulsar-function-go/pf/instanceControlServicer_test.go b/pulsar-function-go/pf/instanceControlServicer_test.go
index 836ec6e5c7940..9344d0a591547 100644
--- a/pulsar-function-go/pf/instanceControlServicer_test.go
+++ b/pulsar-function-go/pf/instanceControlServicer_test.go
@@ -21,6 +21,7 @@ package pf
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"net"
 	"testing"
@@ -76,3 +77,55 @@ func TestInstanceControlServicer_serve_creates_valid_instance(t *testing.T) {
 	log.Printf("Response: %+v", resp.Success)
 	assert.Equal(t, resp.Success, true)
 }
+
+func instanceCommunicationClient(t *testing.T, instance *goInstance) pb.InstanceControlClient {
+	t.Helper()
+
+	if instance == nil {
+		t.Fatalf("cannot create communication client for nil instance")
+	}
+
+	var (
+		ctx context.Context = context.Background()
+		cf  context.CancelFunc
+	)
+
+	if testDeadline, ok := t.Deadline(); ok {
+		ctx, cf = context.WithDeadline(context.Background(), testDeadline)
+		t.Cleanup(cf)
+	}
+
+	lis = bufconn.Listen(bufSize)
+	t.Cleanup(func() {
+		lis.Close()
+	})
+	// create a gRPC server object
+	grpcServer := grpc.NewServer()
+	t.Cleanup(func() {
+		grpcServer.Stop()
+	})
+
+	servicer := InstanceControlServicer{instance}
+	// must register before we start the service.
+	pb.RegisterInstanceControlServer(grpcServer, &servicer)
+
+	// start the server
+	t.Logf("Serving InstanceCommunication on port %d", instance.context.GetPort())
+
+	go func() {
+		if err := grpcServer.Serve(lis); err != nil {
+			panic(fmt.Sprintf("grpc server exited with error: %v", err))
+		}
+	}()
+
+	// Now we can setup the client:
+	conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("Failed to dial bufnet: %v", err)
+	}
+	t.Cleanup(func() {
+		conn.Close()
+	})
+	client := pb.NewInstanceControlClient(conn)
+	return client
+}
diff --git a/pulsar-function-go/pf/stats_test.go b/pulsar-function-go/pf/stats_test.go
index d52b08b717377..7b415ef5eff0b 100644
--- a/pulsar-function-go/pf/stats_test.go
+++ b/pulsar-function-go/pf/stats_test.go
@@ -20,6 +20,7 @@
 package pf
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"math"
@@ -28,6 +29,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 
@@ -257,3 +259,28 @@ func TestUserMetrics(t *testing.T) {
 	gi.close()
 	metricsServicer.close()
 }
+
+func TestInstanceControlMetrics(t *testing.T) {
+	instance := newGoInstance()
+	t.Cleanup(instance.close)
+	instanceClient := instanceCommunicationClient(t, instance)
+	_, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{})
+	assert.NoError(t, err, "err communicating with instance control: %v", err)
+
+	testLabels := []string{"userMetricControlTest1", "userMetricControlTest2"}
+	for _, label := range testLabels {
+		assert.NotContainsf(t, label, "user metrics should not yet contain %s", label)
+	}
+
+	for value, label := range testLabels {
+		instance.context.RecordMetric(label, float64(value+1))
+	}
+	time.Sleep(time.Second)
+
+	metrics, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{})
+	assert.NoError(t, err, "err communicating with instance control: %v", err)
+	for value, label := range testLabels {
+		assert.Containsf(t, metrics.UserMetrics, label, "user metrics should contain metric %s", label)
+		assert.EqualValuesf(t, value+1, metrics.UserMetrics[label], "user metric %s != %d", label, value+1)
+	}
+}

From 4d8156fca0582a350d30557d9243c851c4a45830 Mon Sep 17 00:00:00 2001
From: Christophe Bornet 
Date: Thu, 13 Apr 2023 08:51:52 +0200
Subject: [PATCH 188/404] [improve][ci]Add branch-3.0 to owasp checks (#20081)

---
 .github/workflows/ci-owasp-dependency-check.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index dba1c92ca28dd..8aec501f8d868 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -39,6 +39,7 @@ jobs:
       matrix:
         include:
           - branch: master
+          - branch: branch-3.0
           - branch: branch-2.11
           - branch: branch-2.10
             jdk: 11

From 653271e8d8aeb361ec260780ba88ccf3a880f403 Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Thu, 13 Apr 2023 23:09:10 +0800
Subject: [PATCH 189/404] [fix][test]Fix flaky test produceCommitTest (#20006)

Fixes https://github.com/apache/pulsar/issues/18466
### Motivation
There are two main goals in solving this issue:

1. Fix the unstable tests in `produceCommitTest`.
2. Prevent transaction timeouts created by other tests from affecting the `testTxnTimeoutAtTransactionMetadataStore` test during its execution.


### Modification
1. Change the message-sending method to synchronous. (fix `produceCommitTest`)
2. Increase the transaction timeout to 10 minutes (fix `testTxnTimeoutAtTransactionMetadataStore`).
---
 .../apache/pulsar/client/impl/TransactionEndToEndTest.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java
index 83feaa3ac1158..34cc3bc1ca526 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java
@@ -280,9 +280,9 @@ private void produceCommitTest(boolean enableBatch) throws Exception {
         int messageCnt = 1000;
         for (int i = 0; i < messageCnt; i++) {
             if (i % 5 == 0) {
-                producer.newMessage(txn1).value(("Hello Txn - " + i).getBytes(UTF_8)).send();
+                producer.newMessage(txn1).value(("Hello Txn - " + i).getBytes(UTF_8)).sendAsync();
             } else {
-                producer.newMessage(txn2).value(("Hello Txn - " + i).getBytes(UTF_8)).send();
+                producer.newMessage(txn2).value(("Hello Txn - " + i).getBytes(UTF_8)).sendAsync();
             }
             txnMessageCnt++;
         }
@@ -811,7 +811,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri
     public Transaction getTxn() throws Exception {
         return pulsarClient
                 .newTransaction()
-                .withTransactionTimeout(10, TimeUnit.SECONDS)
+                .withTransactionTimeout(10, TimeUnit.MINUTES)
                 .build()
                 .get();
     }

From 091ee2504ffbe6ec98e354b76e7f4c045e1914aa Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Thu, 13 Apr 2023 10:56:22 -0700
Subject: [PATCH 190/404] [improve][broker] PIP-192 Fix getLastMessageId for
 compressed payload(And add compression and maxBatchSize for the load balance
 system topic) (#20087)

---
 .../channel/ServiceUnitStateChannelImpl.java  |  4 ++
 .../pulsar/broker/service/BrokerService.java  |  6 +-
 .../pulsar/broker/service/ServerCnx.java      | 11 ++++
 .../impl/RawBatchMessageContainerImpl.java    | 12 ++--
 .../StrategicTwoPhaseCompactor.java           | 17 ++++-
 .../RawBatchMessageContainerImplTest.java     | 63 +++++++++++++------
 .../GetLastMessageIdCompactedTest.java        | 43 +++++++++++++
 .../ServiceUnitStateCompactionTest.java       | 24 +++++--
 8 files changed, 145 insertions(+), 35 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index bd62c53be6083..68c6440e68e4b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -81,6 +81,7 @@
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClientException;
@@ -106,6 +107,8 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
             TopicDomain.persistent.value(),
             SYSTEM_NAMESPACE,
             "loadbalancer-service-unit-state").toString();
+
+    public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD;
     private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec
     public static final long VERSION_ID_INIT = 1; // initial versionId
     private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60;
@@ -285,6 +288,7 @@ public synchronized void start() throws PulsarServerException {
 
             producer = pulsar.getClient().newProducer(schema)
                     .enableBatching(true)
+                    .compressionType(MSG_COMPRESSION_TYPE)
                     .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES)
                     .blockIfQueueFull(true)
                     .topic(TOPIC)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index bb08734298f04..33e5500d623d2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -2144,9 +2144,9 @@ public CompletableFuture checkTopicNsOwnership(final String topic) {
                     if (ownedByThisInstance) {
                         return CompletableFuture.completedFuture(null);
                     } else {
-                        String msg = String.format("Namespace bundle for topic (%s) not served by this instance. "
-                                        + "Please redo the lookup. Request is denied: namespace=%s", topic,
-                                topicName.getNamespace());
+                        String msg = String.format("Namespace bundle for topic (%s) not served by this instance:%s. "
+                                        + "Please redo the lookup. Request is denied: namespace=%s",
+                                topic, pulsar.getLookupServiceAddress(), topicName.getNamespace());
                         log.warn(msg);
                         return FutureUtil.failedFuture(new ServiceUnitNotReadyException(msg));
                     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 01274ab4460cb..888668e15b167 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -129,6 +129,7 @@
 import org.apache.pulsar.common.api.proto.CommandUnsubscribe;
 import org.apache.pulsar.common.api.proto.CommandWatchTopicList;
 import org.apache.pulsar.common.api.proto.CommandWatchTopicListClose;
+import org.apache.pulsar.common.api.proto.CompressionType;
 import org.apache.pulsar.common.api.proto.FeatureFlags;
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
 import org.apache.pulsar.common.api.proto.KeySharedMode;
@@ -141,6 +142,8 @@
 import org.apache.pulsar.common.api.proto.ServerError;
 import org.apache.pulsar.common.api.proto.SingleMessageMetadata;
 import org.apache.pulsar.common.api.proto.TxnAction;
+import org.apache.pulsar.common.compression.CompressionCodec;
+import org.apache.pulsar.common.compression.CompressionCodecProvider;
 import org.apache.pulsar.common.intercept.InterceptException;
 import org.apache.pulsar.common.naming.Metadata;
 import org.apache.pulsar.common.naming.NamespaceName;
@@ -2167,6 +2170,14 @@ private int calculateTheLastBatchIndexInBatch(MessageMetadata metadata, ByteBuf
         if (batchSize <= 1){
             return -1;
         }
+        if (metadata.hasCompression()) {
+            var tmp = payload;
+            CompressionType compressionType = metadata.getCompression();
+            CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
+            int uncompressedSize = metadata.getUncompressedSize();
+            payload = codec.decode(payload, uncompressedSize);
+            tmp.release();
+        }
         SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
         int lastBatchIndexInBatch = -1;
         for (int i = 0; i < batchSize; i++){
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java
index cf6b213155cd7..7e1c2cd5e3fe3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java
@@ -47,13 +47,13 @@ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl {
     MessageCrypto msgCrypto;
     Set encryptionKeys;
     CryptoKeyReader cryptoKeyReader;
-    public RawBatchMessageContainerImpl(int maxNumMessagesInBatch) {
+
+    public RawBatchMessageContainerImpl(int maxNumMessagesInBatch, int maxBytesInBatch) {
         super();
-        compressionType = CompressionType.NONE;
-        compressor = new CompressionCodecNone();
-        if (maxNumMessagesInBatch > 0) {
-            this.maxNumMessagesInBatch = maxNumMessagesInBatch;
-        }
+        this.compressionType = CompressionType.NONE;
+        this.compressor = new CompressionCodecNone();
+        this.maxNumMessagesInBatch = maxNumMessagesInBatch;
+        this.maxBytesInBatch = maxBytesInBatch;
     }
     private ByteBuf encrypt(ByteBuf compressedPayload) {
         if (msgCrypto == null) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
index 37b03e275d6bf..557d4a6580161 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.compaction;
 
+import com.google.common.annotations.VisibleForTesting;
 import io.netty.buffer.ByteBuf;
 import java.time.Duration;
 import java.util.Iterator;
@@ -62,17 +63,29 @@
 public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor {
     private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class);
     private static final int MAX_OUTSTANDING = 500;
+    private static final int MAX_NUM_MESSAGES_IN_BATCH = 1000;
+    private static final int MAX_BYTES_IN_BATCH = 128 * 1024;
     private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000;
     private final Duration phaseOneLoopReadTimeout;
     private final RawBatchMessageContainerImpl batchMessageContainer;
 
+    @VisibleForTesting
     public StrategicTwoPhaseCompactor(ServiceConfiguration conf,
                                       PulsarClient pulsar,
                                       BookKeeper bk,
                                       ScheduledExecutorService scheduler,
                                       int maxNumMessagesInBatch) {
+        this(conf, pulsar, bk, scheduler, maxNumMessagesInBatch, MAX_BYTES_IN_BATCH);
+    }
+
+    private StrategicTwoPhaseCompactor(ServiceConfiguration conf,
+                                      PulsarClient pulsar,
+                                      BookKeeper bk,
+                                      ScheduledExecutorService scheduler,
+                                      int maxNumMessagesInBatch,
+                                      int maxBytesInBatch) {
         super(conf, pulsar, bk, scheduler);
-        batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch);
+        batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch, maxBytesInBatch);
         phaseOneLoopReadTimeout = Duration.ofSeconds(conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds());
     }
 
@@ -80,7 +93,7 @@ public StrategicTwoPhaseCompactor(ServiceConfiguration conf,
                                       PulsarClient pulsar,
                                       BookKeeper bk,
                                       ScheduledExecutorService scheduler) {
-        this(conf, pulsar, bk, scheduler, -1);
+        this(conf, pulsar, bk, scheduler, MAX_NUM_MESSAGES_IN_BATCH, MAX_BYTES_IN_BATCH);
     }
 
     public CompletableFuture compact(String topic) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java
index 9fa834a166cda..9b8b1e5efb99c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java
@@ -19,8 +19,10 @@
 package org.apache.pulsar.client.impl;
 
 
-import static org.apache.pulsar.common.api.proto.CompressionType.LZ4;
 import static org.apache.pulsar.common.api.proto.CompressionType.NONE;
+import static org.apache.pulsar.common.api.proto.CompressionType.ZSTD;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import java.io.IOException;
@@ -45,6 +47,7 @@
 import org.apache.pulsar.compaction.CompactionTest;
 import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class RawBatchMessageContainerImplTest {
@@ -53,9 +56,11 @@ public class RawBatchMessageContainerImplTest {
     CryptoKeyReader cryptoKeyReader;
     Map encryptKeys;
 
+    int maxBytesInBatch = 5 * 1024 * 1024;
+
     public void setEncryptionAndCompression(boolean encrypt, boolean compress) {
         if (compress) {
-            compressionType = LZ4;
+            compressionType = ZSTD;
         } else {
             compressionType = NONE;
         }
@@ -100,14 +105,24 @@ public MessageImpl createMessage(String topic, String value, int entryId) {
 
     @BeforeMethod
     public void setup() throws Exception {
-        setEncryptionAndCompression(false, false);
+        setEncryptionAndCompression(false, true);
     }
-    @Test
-    public void testToByteBuf() throws IOException {
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2);
+    @DataProvider(name = "testBatchLimitByMessageCount")
+    public static Object[][] testBatchLimitByMessageCount() {
+        return new Object[][] {{true}, {false}};
+    }
+
+    @Test(timeOut = 20000, dataProvider = "testBatchLimitByMessageCount")
+    public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) throws IOException {
+        RawBatchMessageContainerImpl container = testBatchLimitByMessageCount ?
+                new RawBatchMessageContainerImpl(2, Integer.MAX_VALUE) :
+                new RawBatchMessageContainerImpl(Integer.MAX_VALUE, 5);
+
         String topic = "my-topic";
-        container.add(createMessage(topic, "hi-1", 0), null);
-        container.add(createMessage(topic, "hi-2", 1), null);
+        var full1 = container.add(createMessage(topic, "hi-1", 0), null);
+        var full2 = container.add(createMessage(topic, "hi-2", 1), null);
+        assertFalse(full1);
+        assertTrue(full2);
         ByteBuf buf = container.toByteBuf();
 
 
@@ -126,18 +141,23 @@ public void testToByteBuf() throws IOException {
         MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder();
         Assert.assertEquals(metadata.getNumMessagesInBatch(), 2);
         Assert.assertEquals(metadata.getHighestSequenceId(), 1);
-        Assert.assertEquals(metadata.getCompression(), NONE);
+        Assert.assertEquals(metadata.getCompression(), ZSTD);
+
+        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
+        ByteBuf payload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
 
         SingleMessageMetadata messageMetadata = new SingleMessageMetadata();
+        messageMetadata.setCompactedOut(true);
         ByteBuf payload1 = Commands.deSerializeSingleMessageInBatch(
-                singleMessageMetadataAndPayload.getPayload(), messageMetadata, 0, 2);
+                payload, messageMetadata, 0, 2);
         ByteBuf payload2 = Commands.deSerializeSingleMessageInBatch(
-                singleMessageMetadataAndPayload.getPayload(), messageMetadata, 1, 2);
+                payload, messageMetadata, 1, 2);
 
         Assert.assertEquals(payload1.toString(Charset.defaultCharset()), "hi-1");
         Assert.assertEquals(payload2.toString(Charset.defaultCharset()), "hi-2");
         payload1.release();
         payload2.release();
+        payload.release();
         singleMessageMetadataAndPayload.release();
         metadataAndPayload.release();
         buf.release();
@@ -147,7 +167,7 @@ public void testToByteBuf() throws IOException {
     public void testToByteBufWithCompressionAndEncryption() throws IOException {
         setEncryptionAndCompression(true, true);
 
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2);
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch);
         container.setCryptoKeyReader(cryptoKeyReader);
         String topic = "my-topic";
         container.add(createMessage(topic, "hi-1", 0), null);
@@ -169,7 +189,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException {
         MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder();
         Assert.assertEquals(metadata.getNumMessagesInBatch(), 2);
         Assert.assertEquals(metadata.getHighestSequenceId(), 1);
-        Assert.assertEquals(metadata.getCompression(), compressionType);
+        Assert.assertEquals(metadata.getCompression(), ZSTD);
 
         ByteBuf payload = singleMessageMetadataAndPayload.getPayload();
         int maxDecryptedSize = msgCrypto.getMaxOutputSize(payload.readableBytes());
@@ -197,7 +217,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException {
 
     @Test
     public void testToByteBufWithSingleMessage() throws IOException {
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2);
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch);
         String topic = "my-topic";
         container.add(createMessage(topic, "hi-1", 0), null);
         ByteBuf buf = container.toByteBuf();
@@ -218,9 +238,12 @@ public void testToByteBufWithSingleMessage() throws IOException {
         MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder();
         Assert.assertEquals(metadata.getNumMessagesInBatch(), 1);
         Assert.assertEquals(metadata.getHighestSequenceId(), 0);
-        Assert.assertEquals(metadata.getCompression(), NONE);
+        Assert.assertEquals(metadata.getCompression(), ZSTD);
+
+        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
+        ByteBuf payload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
 
-        Assert.assertEquals(singleMessageMetadataAndPayload.getPayload().toString(Charset.defaultCharset()), "hi-1");
+        Assert.assertEquals(payload.toString(Charset.defaultCharset()), "hi-1");
         singleMessageMetadataAndPayload.release();
         metadataAndPayload.release();
         buf.release();
@@ -228,7 +251,7 @@ public void testToByteBufWithSingleMessage() throws IOException {
 
     @Test
     public void testMaxNumMessagesInBatch() {
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1);
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch);
         String topic = "my-topic";
 
         boolean isFull = container.add(createMessage(topic, "hi", 0), null);
@@ -238,14 +261,14 @@ public void testMaxNumMessagesInBatch() {
 
     @Test(expectedExceptions = UnsupportedOperationException.class)
     public void testCreateOpSendMsg() {
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1);
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch);
         container.createOpSendMsg();
     }
 
     @Test
     public void testToByteBufWithEncryptionWithoutCryptoKeyReader() {
         setEncryptionAndCompression(true, false);
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1);
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch);
         String topic = "my-topic";
         container.add(createMessage(topic, "hi-1", 0), null);
         Assert.assertEquals(container.getNumMessagesInBatch(), 1);
@@ -263,7 +286,7 @@ public void testToByteBufWithEncryptionWithoutCryptoKeyReader() {
     @Test
     public void testToByteBufWithEncryptionWithInvalidEncryptKeys() {
         setEncryptionAndCompression(true, false);
-        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1);
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch);
         container.setCryptoKeyReader(cryptoKeyReader);
         encryptKeys = new HashMap<>();
         encryptKeys.put(null, null);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java
index 0be9fa407542f..317b1a227e585 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java
@@ -30,6 +30,7 @@
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.Producer;
@@ -263,6 +264,48 @@ public void testGetLastMessageIdAfterCompaction(boolean enabledBatch) throws Exc
         admin.topics().delete(topicName, false);
     }
 
+    @Test(dataProvider = "enabledBatch")
+    public void testGetLastMessageIdAfterCompactionWithCompression(boolean enabledBatch) throws Exception {
+        String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp");
+        String subName = "sub";
+        Consumer consumer = createConsumer(topicName, subName);
+        var producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .batchingMaxPublishDelay(3, TimeUnit.HOURS)
+                .batchingMaxBytes(Integer.MAX_VALUE)
+                .compressionType(CompressionType.ZSTD)
+                .enableBatching(enabledBatch).create();
+
+        List> sendFutures = new ArrayList<>();
+        sendFutures.add(producer.newMessage().key("k0").value("v0").sendAsync());
+        sendFutures.add(producer.newMessage().key("k0").value("v1").sendAsync());
+        sendFutures.add(producer.newMessage().key("k0").value("v2").sendAsync());
+        producer.flush();
+        sendFutures.add(producer.newMessage().key("k1").value("v0").sendAsync());
+        sendFutures.add(producer.newMessage().key("k1").value("v1").sendAsync());
+        sendFutures.add(producer.newMessage().key("k1").value("v2").sendAsync());
+        producer.flush();
+        FutureUtil.waitForAll(sendFutures).join();
+
+        triggerCompactionAndWait(topicName);
+
+        MessageIdImpl lastMessageIdByTopic = getLastMessageIdByTopic(topicName);
+        MessageIdImpl messageId = (MessageIdImpl) consumer.getLastMessageId();
+        assertEquals(messageId.getLedgerId(), lastMessageIdByTopic.getLedgerId());
+        assertEquals(messageId.getEntryId(), lastMessageIdByTopic.getEntryId());
+        if (enabledBatch){
+            BatchMessageIdImpl lastBatchMessageIdByTopic = (BatchMessageIdImpl) lastMessageIdByTopic;
+            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
+            assertEquals(batchMessageId.getBatchSize(), lastBatchMessageIdByTopic.getBatchSize());
+            assertEquals(batchMessageId.getBatchIndex(), lastBatchMessageIdByTopic.getBatchIndex());
+        }
+
+        // cleanup.
+        consumer.close();
+        producer.close();
+        admin.topics().delete(topicName, false);
+    }
+
     @Test(dataProvider = "enabledBatch")
     public void testGetLastMessageIdAfterCompactionEndWithNullMsg(boolean enabledBatch) throws Exception {
         String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java
index 02812898dc49a..1a69a86f7c6b2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java
@@ -25,6 +25,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition;
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
@@ -185,6 +186,7 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .messageRoutingMode(MessageRoutingMode.SinglePartition)
                 .create();
@@ -352,12 +354,13 @@ public void testCompactionWithTableview() throws Exception {
         compactor.compact(topic, strategy).get();
 
         // consumer with readCompacted enabled only get compacted entries
-        var tableview = pulsar.getClient().newTableViewBuilder(schema)
+        var tableview = pulsar.getClient().newTableView(schema)
                 .topic(topic)
                 .loadConf(Map.of(
                         "topicCompactionStrategyClassName",
                         ServiceUnitStateCompactionStrategy.class.getName()))
                 .create();
+
         for(var etr : tableview.entrySet()){
             Assert.assertEquals(expected.remove(etr.getKey()), etr.getValue());
             if (expected.isEmpty()) {
@@ -376,6 +379,7 @@ public void testReadCompactedBeforeCompaction() throws Exception {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .create();
 
@@ -420,6 +424,7 @@ public void testReadEntriesAfterCompaction() throws Exception {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .create();
 
@@ -460,6 +465,7 @@ public void testSeekEarliestAfterCompaction() throws Exception {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .create();
 
@@ -557,6 +563,7 @@ public void testSlowTableviewAfterCompaction() throws Exception {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .messageRoutingMode(MessageRoutingMode.SinglePartition)
                 .create();
@@ -627,6 +634,7 @@ public void testBrokerRestartAfterCompaction() throws Exception {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .create();
         String key = "key0";
@@ -672,6 +680,7 @@ public void testCompactEmptyTopic() throws Exception {
 
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
                 .enableBatching(true)
                 .create();
 
@@ -733,7 +742,9 @@ public void testWholeBatchCompactedOut() throws Exception {
     public void testCompactionWithLastDeletedKey() throws Exception {
         String topic = "persistent://my-property/use/my-ns/my-topic1";
 
-        Producer producer = pulsarClient.newProducer(schema).topic(topic).enableBatching(true)
+        Producer producer = pulsarClient.newProducer(schema).topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
+                .enableBatching(true)
                 .messageRoutingMode(MessageRoutingMode.SinglePartition).create();
 
         pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
@@ -761,7 +772,9 @@ public void testCompactionWithLastDeletedKey() throws Exception {
     public void testEmptyCompactionLedger() throws Exception {
         String topic = "persistent://my-property/use/my-ns/my-topic1";
 
-        Producer producer = pulsarClient.newProducer(schema).topic(topic).enableBatching(true)
+        Producer producer = pulsarClient.newProducer(schema).topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
+                .enableBatching(true)
                 .messageRoutingMode(MessageRoutingMode.SinglePartition).create();
 
         pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
@@ -791,7 +804,8 @@ public void testAllEmptyCompactionLedger() throws Exception {
         final int messages = 10;
 
         // 1.create producer and publish message to the topic.
-        ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic);
+        ProducerBuilder builder = pulsarClient.newProducer(schema)
+                .compressionType(MSG_COMPRESSION_TYPE).topic(topic);
         builder.batchingMaxMessages(messages / 5);
 
         Producer producer = builder.create();
@@ -828,6 +842,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage()
 
         // 1.create producer and publish message to the topic.
         ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic);
+        builder.compressionType(MSG_COMPRESSION_TYPE);
         builder.enableBatching(true);
 
 
@@ -876,6 +891,7 @@ public void testReadUnCompacted()
 
         // 1.create producer and publish message to the topic.
         ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic);
+        builder.compressionType(MSG_COMPRESSION_TYPE);
         builder.batchingMaxMessages(messages / 5);
 
         Producer producer = builder.create();

From 6152cc82ac8a344cfc2539c56f15cec6f4a1cff7 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sat, 15 Apr 2023 13:42:07 +0800
Subject: [PATCH 191/404] [fix][broker] Fix avoid future of clear delayed
 message can't complete (#20075)

---
 .../BucketDelayedDeliveryTrackerFactory.java  | 28 +++++++++++
 .../pulsar/broker/delayed/bucket/Bucket.java  |  3 +-
 .../bucket/BucketDelayedDeliveryTracker.java  |  4 +-
 ...PersistentDispatcherMultipleConsumers.java | 17 +++----
 .../service/persistent/PersistentTopic.java   | 26 +++++-----
 .../persistent/BucketDelayedDeliveryTest.java | 48 +++++++++++++++++++
 6 files changed, 102 insertions(+), 24 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
index f0feb8b27d6a1..6a00bfd199584 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
@@ -21,13 +21,19 @@
 import io.netty.util.HashedWheelTimer;
 import io.netty.util.Timer;
 import io.netty.util.concurrent.DefaultThreadFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
+import org.apache.pulsar.common.util.FutureUtil;
 
 public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrackerFactory {
 
@@ -72,6 +78,28 @@ public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers d
                 delayedDeliveryMaxIndexesPerBucketSnapshotSegment, delayedDeliveryMaxNumBuckets);
     }
 
+    /**
+     * Clean up residual snapshot data.
+     * If tracker has not been created or has been closed, then we can't clean up the snapshot with `tracker.clear`,
+     * this method can clean up the residual snapshots without creating a tracker.
+     */
+    public CompletableFuture cleanResidualSnapshots(ManagedCursor cursor) {
+        Map cursorProperties = cursor.getCursorProperties();
+        List> futures = new ArrayList<>();
+        FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create();
+        cursorProperties.forEach((k, v) -> {
+            if (k != null && v != null && k.startsWith(BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX)) {
+                CompletableFuture future = sequencer.sequential(() -> {
+                    return cursor.removeCursorProperty(k)
+                            .thenCompose(__ -> bucketSnapshotStorage.deleteBucketSnapshot(Long.parseLong(v)));
+                });
+                futures.add(future);
+            }
+        });
+
+        return FutureUtil.waitForAll(futures);
+    }
+
     @Override
     public void close() throws Exception {
         if (bucketSnapshotStorage != null) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index 5b7023be5034e..4d7d3aa512be6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -18,8 +18,8 @@
  */
 package org.apache.pulsar.broker.delayed.bucket;
 
-import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
+import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,7 +41,6 @@
 @AllArgsConstructor
 abstract class Bucket {
 
-    static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket";
     static final String DELIMITER = "_";
     static final int MaxRetryTimes = 3;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index a34bd51af98e4..6ead1e207b0ce 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -19,7 +19,7 @@
 package org.apache.pulsar.broker.delayed.bucket;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELAYED_BUCKET_KEY_PREFIX;
+import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
 import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER;
 import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes;
 import com.google.common.annotations.VisibleForTesting;
@@ -66,6 +66,8 @@
 @ThreadSafe
 public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker {
 
+    public static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket";
+
     static final CompletableFuture NULL_LONG_PROMISE = CompletableFuture.completedFuture(null);
 
     static final int AsyncOperationTimeoutSeconds = 60;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 4ac755860fc7b..c60b4562bf111 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -49,6 +49,7 @@
 import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
+import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
 import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers;
@@ -1098,19 +1099,15 @@ public CompletableFuture clearDelayedMessages() {
             return CompletableFuture.completedFuture(null);
         }
 
-        if (delayedDeliveryTracker.isEmpty() && topic.getBrokerService()
-                .getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory) {
-            synchronized (this) {
-                if (delayedDeliveryTracker.isEmpty()) {
-                    delayedDeliveryTracker = Optional
-                            .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this));
-                }
-            }
-        }
-
         if (delayedDeliveryTracker.isPresent()) {
             return this.delayedDeliveryTracker.get().clear();
         } else {
+            DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory =
+                    topic.getBrokerService().getDelayedDeliveryTrackerFactory();
+            if (delayedDeliveryTrackerFactory instanceof BucketDelayedDeliveryTrackerFactory
+                    bucketDelayedDeliveryTrackerFactory) {
+                return bucketDelayedDeliveryTrackerFactory.cleanResidualSnapshots(cursor);
+            }
             return CompletableFuture.completedFuture(null);
         }
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 0f5e60439810f..fe181bb1c01f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -81,6 +81,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
+import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy;
 import org.apache.pulsar.broker.namespace.NamespaceService;
@@ -1169,15 +1170,21 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName,
         }
 
         Dispatcher dispatcher = persistentSubscription.getDispatcher();
-        final Dispatcher temporaryDispatcher;
         if (dispatcher == null) {
-            log.info("[{}][{}] Dispatcher is null, try to create temporary dispatcher to clear delayed message", topic,
-                    subscriptionName);
-            dispatcher = temporaryDispatcher =
-                    new PersistentDispatcherMultipleConsumers(this, persistentSubscription.cursor,
-                            persistentSubscription);
-        } else {
-            temporaryDispatcher = null;
+            DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory =
+                    brokerService.getDelayedDeliveryTrackerFactory();
+            if (delayedDeliveryTrackerFactory instanceof BucketDelayedDeliveryTrackerFactory
+                    bucketDelayedDeliveryTrackerFactory) {
+                ManagedCursor cursor = persistentSubscription.getCursor();
+                bucketDelayedDeliveryTrackerFactory.cleanResidualSnapshots(cursor).whenComplete((__, ex) -> {
+                    if (ex != null) {
+                        unsubscribeFuture.completeExceptionally(ex);
+                    } else {
+                        asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+                    }
+                });
+            }
+            return;
         }
 
         dispatcher.clearDelayedMessages().whenComplete((__, ex) -> {
@@ -1186,9 +1193,6 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName,
             } else {
                 asyncDeleteCursor(subscriptionName, unsubscribeFuture);
             }
-            if (temporaryDispatcher != null) {
-                temporaryDispatcher.close();
-            }
         });
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 5480a2e7a704a..0a82b2b4c3cb0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -305,4 +305,52 @@ public void testBucketDelayedIndexMetrics() throws Exception {
         assertTrue(namespaceMetric.isPresent());
         assertEquals(6, namespaceMetric.get().value);
     }
+
+    @Test
+    public void testDelete() throws Exception {
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testDelete");
+
+        @Cleanup
+        Consumer c1 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        for (int i = 0; i < 1000; i++) {
+            producer.newMessage()
+                    .value("msg")
+                    .deliverAfter(1, TimeUnit.HOURS)
+                    .send();
+        }
+
+        Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000));
+
+        Map cursorProperties =
+                ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties();
+        List bucketIds = cursorProperties.entrySet().stream()
+                .filter(x -> x.getKey().startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).map(
+                        x -> Long.valueOf(x.getValue())).toList();
+
+        assertTrue(bucketIds.size() > 0);
+
+        admin.topics().delete(topic, true);
+
+        for (Long bucketId : bucketIds) {
+            try {
+                LedgerHandle ledgerHandle =
+                        pulsarTestContext.getBookKeeperClient()
+                                .openLedger(bucketId, BookKeeper.DigestType.CRC32C, new byte[]{});
+                Assert.fail("Should fail");
+            } catch (BKException.BKNoSuchLedgerExistsException e) {
+                // ignore it
+            }
+        }
+    }
 }

From c4aec6661e795c46181dc1fa79282065fa875768 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sun, 16 Apr 2023 16:05:44 +0800
Subject: [PATCH 192/404] [fix][broker] Ensure previous delayed index be
 removed from snapshotSegmentLastIndexTable & Make load operate asynchronous
 (#20086)

---
 .../bucket/BucketDelayedDeliveryTracker.java  | 106 ++++++++++--------
 ...PersistentDispatcherMultipleConsumers.java |  13 +--
 .../BucketDelayedDeliveryTrackerTest.java     |  30 +++--
 3 files changed, 85 insertions(+), 64 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 6ead1e207b0ce..6678c6df254b0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -21,7 +21,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
 import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER;
-import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.Range;
@@ -84,7 +83,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final int maxNumBuckets;
 
-    private long numberDelayedMessages;
+    private volatile long numberDelayedMessages;
 
     @Getter
     @VisibleForTesting
@@ -102,6 +101,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final BucketDelayedMessageIndexStats stats;
 
+    private CompletableFuture pendingLoad = null;
+
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
@@ -269,7 +270,7 @@ private void afterCreateImmutableBucket(Pair immu
                     if (ex == null) {
                         immutableBucket.setSnapshotSegments(null);
                         immutableBucket.asyncUpdateSnapshotLength();
-                        log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(),
+                        log.info("[{}] Create bucket snapshot finish, bucketKey: {}", dispatcher.getName(),
                                 immutableBucket.bucketKey());
 
                         stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create,
@@ -529,17 +530,25 @@ protected long nextDeliveryTime() {
     }
 
     @Override
-    public synchronized long getNumberOfDelayedMessages() {
+    public long getNumberOfDelayedMessages() {
         return numberDelayedMessages;
     }
 
     @Override
-    public synchronized long getBufferMemoryUsage() {
+    public long getBufferMemoryUsage() {
         return this.lastMutableBucket.getBufferMemoryUsage() + sharedBucketPriorityQueue.bytesCapacity();
     }
 
     @Override
     public synchronized NavigableSet getScheduledMessages(int maxMessages) {
+        if (!checkPendingOpDone()) {
+            if (log.isDebugEnabled()) {
+                log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.",
+                        dispatcher.getName());
+            }
+            return Collections.emptyNavigableSet();
+        }
+
         long cutoffTime = getCutoffTime();
 
         lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue);
@@ -558,6 +567,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
             ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId);
             if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) {
+                // All message of current snapshot segment are scheduled, try load next snapshot segment
                 if (bucket.merging) {
                     log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}",
                             dispatcher.getName(), bucket.bucketKey());
@@ -569,26 +579,19 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                     log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}",
                             dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1);
                 }
-                // All message of current snapshot segment are scheduled, load next snapshot segment
-                // TODO make it asynchronous and not blocking this process
-                try {
-                    boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone();
-
-                    if (!createFutureDone) {
-                        log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}",
-                                dispatcher.getName(), bucket.bucketKey());
-                        break;
-                    }
-
-                    if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) {
-                        immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
-                        bucket.asyncDeleteBucketSnapshot(stats);
-                        continue;
-                    }
+                boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone();
+                if (!createFutureDone) {
+                    log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}",
+                            dispatcher.getName(), bucket.bucketKey());
+                    break;
+                }
 
-                    long loadStartTime = System.currentTimeMillis();
-                    stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load);
-                    bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> {
+                long loadStartTime = System.currentTimeMillis();
+                stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load);
+                CompletableFuture loadFuture = pendingLoad = bucket.asyncLoadNextBucketSnapshotEntry()
+                        .thenAccept(indexList -> {
+                    synchronized (BucketDelayedDeliveryTracker.this) {
+                        this.snapshotSegmentLastIndexTable.remove(ledgerId, entryId);
                         if (CollectionUtils.isEmpty(indexList)) {
                             immutableBuckets.asMapOfRanges()
                                     .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
@@ -603,31 +606,36 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                             sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(),
                                     index.getEntryId());
                         }
-                    }).whenComplete((__, ex) -> {
-                        if (ex != null) {
-                            // Back bucket state
-                            bucket.setCurrentSegmentEntryId(preSegmentEntryId);
-
-                            log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}",
-                                    dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex);
-
-                            stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load);
-                        } else {
-                            log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}",
-                                    dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1);
-
-                            stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load,
-                                    System.currentTimeMillis() - loadStartTime);
+                    }
+                }).whenComplete((__, ex) -> {
+                    if (ex != null) {
+                        // Back bucket state
+                        bucket.setCurrentSegmentEntryId(preSegmentEntryId);
+
+                        log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}",
+                                dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex);
+
+                        stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load);
+                    } else {
+                        log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}",
+                                dispatcher.getName(), bucket.bucketKey(),
+                                (preSegmentEntryId == bucket.lastSegmentEntryId) ? "-1" : preSegmentEntryId + 1);
+
+                        stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load,
+                                System.currentTimeMillis() - loadStartTime);
+                    }
+                    synchronized (this) {
+                        if (timeout != null) {
+                            timeout.cancel();
                         }
-                    }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    // Ignore exception to reload this segment on the next schedule.
-                    log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}",
-                            dispatcher.getName(), bucket.bucketKey(), e);
+                        timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS);
+                    }
+                });
+
+                if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) {
                     break;
                 }
             }
-            snapshotSegmentLastIndexTable.remove(ledgerId, entryId);
 
             positions.add(new PositionImpl(ledgerId, entryId));
 
@@ -643,6 +651,14 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
         return positions;
     }
 
+    private synchronized boolean checkPendingOpDone() {
+        if (pendingLoad == null || pendingLoad.isDone()) {
+            pendingLoad = null;
+            return true;
+        }
+        return false;
+    }
+
     @Override
     public boolean shouldPauseAllDeliveries() {
         return false;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index c60b4562bf111..81adda053e8cd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -46,7 +46,6 @@
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory;
@@ -1089,7 +1088,7 @@ protected synchronized boolean shouldPauseDeliveryForDelayTracker() {
     }
 
     @Override
-    public synchronized long getNumberOfDelayedMessages() {
+    public long getNumberOfDelayedMessages() {
         return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L);
     }
 
@@ -1169,15 +1168,7 @@ public PersistentTopic getTopic() {
 
 
     public long getDelayedTrackerMemoryUsage() {
-        if (delayedDeliveryTracker.isEmpty()) {
-            return 0;
-        }
-
-        if (delayedDeliveryTracker.get() instanceof AbstractDelayedDeliveryTracker) {
-            return delayedDeliveryTracker.get().getBufferMemoryUsage();
-        }
-
-        return 0;
+        return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L);
     }
 
     public Map getBucketDelayedIndexStats() {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 95234d688f6a0..39b3992fbd195 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -39,6 +39,7 @@
 import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -197,9 +198,11 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
         });
 
         assertTrue(tracker.hasMessageAvailable());
-        Set scheduledMessages = tracker.getScheduledMessages(100);
-
-        assertEquals(scheduledMessages.size(), 1);
+        Set scheduledMessages = new TreeSet<>();
+        Awaitility.await().untilAsserted(() -> {
+            scheduledMessages.addAll(tracker.getScheduledMessages(100));
+            assertEquals(scheduledMessages.size(), 1);
+        });
 
         tracker.addMessage(101, 101, 101 * 10);
 
@@ -216,12 +219,15 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
         clockTime.set(100 * 10);
 
         assertTrue(tracker2.hasMessageAvailable());
-        scheduledMessages = tracker2.getScheduledMessages(70);
+        Set scheduledMessages2 = new TreeSet<>();
 
-        assertEquals(scheduledMessages.size(), 70);
+        Awaitility.await().untilAsserted(() -> {
+            scheduledMessages2.addAll(tracker2.getScheduledMessages(70));
+            assertEquals(scheduledMessages2.size(), 70);
+        });
 
         int i = 31;
-        for (PositionImpl scheduledMessage : scheduledMessages) {
+        for (PositionImpl scheduledMessage : scheduledMessages2) {
             assertEquals(scheduledMessage, PositionImpl.get(i, i));
             i++;
         }
@@ -298,7 +304,11 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) {
 
         clockTime.set(110 * 10);
 
-        NavigableSet scheduledMessages = tracker2.getScheduledMessages(110);
+        NavigableSet scheduledMessages = new TreeSet<>();
+        Awaitility.await().untilAsserted(() -> {
+            scheduledMessages.addAll(tracker2.getScheduledMessages(110));
+            assertEquals(scheduledMessages.size(), 110);
+        });
         for (int i = 1; i <= 110; i++) {
             PositionImpl position = scheduledMessages.pollFirst();
             assertEquals(position, PositionImpl.get(i, i));
@@ -370,7 +380,11 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) {
 
         assertEquals(tracker2.getScheduledMessages(100).size(), 0);
 
-        assertEquals(tracker2.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue);
+        Set scheduledMessages = new TreeSet<>();
+        Awaitility.await().untilAsserted(() -> {
+            scheduledMessages.addAll(tracker2.getScheduledMessages(100));
+            assertEquals(scheduledMessages.size(), delayedMessagesInSnapshotValue);
+        });
 
         assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty());

From 47b24b071353f9285704e29e397b851e31df4037 Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Sun, 16 Apr 2023 17:05:46 +0300
Subject: [PATCH 193/404] [fix][doc] Add link to PIP proposal (#19911)

### Motivation

Add a link to see existing PIP issues so you can select a number
---
 wiki/proposals/PIP.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md
index e10452b107fd8..159c9199895d1 100644
--- a/wiki/proposals/PIP.md
+++ b/wiki/proposals/PIP.md
@@ -80,7 +80,7 @@ The process works in the following way:
 1. The author(s) of the proposal will create a GitHub issue ticket choosing the
    template for PIP proposals. The issue title should be "PIP-xxx: title", where
    the "xxx" number should be chosen to be the next number from the existing PIP 
-   issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)).
+   issues, listed [here](https://github.com/apache/pulsar/issues?q=is%3Aissue+label%3APIP+)
 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list
    to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion
    need to happen in the mailing list. Please avoid discussing it using

From b7eab9469177eda2c56e36bb9871aab48a17d4ec Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 17 Apr 2023 10:59:32 +0800
Subject: [PATCH 194/404] [fix][doc] ConnectorDocGenerator support Java 9+
 (#20100)

Signed-off-by: tison 
---
 .../pulsar/io/docs/ConnectorDocGenerator.java | 69 +++++++------------
 .../io/elasticsearch/ElasticSearchConfig.java |  5 ++
 2 files changed, 31 insertions(+), 43 deletions(-)

diff --git a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
index fe12b2b11ce84..fec7b12087977 100644
--- a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
+++ b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
@@ -21,6 +21,7 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.google.common.base.Strings;
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
@@ -28,11 +29,9 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
+import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
@@ -47,21 +46,12 @@ public class ConnectorDocGenerator {
     private static final String INDENT = "  ";
 
     private static Reflections newReflections() throws Exception {
-        List urls = new ArrayList<>();
-        ClassLoader[] classLoaders = new ClassLoader[] {
-            ConnectorDocGenerator.class.getClassLoader(),
-            Thread.currentThread().getContextClassLoader()
-        };
-        for (int i = 0; i < classLoaders.length; i++) {
-            if (classLoaders[i] instanceof URLClassLoader) {
-                urls.addAll(Arrays.asList(((URLClassLoader) classLoaders[i]).getURLs()));
-            } else {
-                throw new RuntimeException("ClassLoader '" + classLoaders[i] + " is not an instance of URLClassLoader");
-            }
+        final String[] classpathList = System.getProperty("java.class.path").split(":");
+        final List urlList = new ArrayList<>();
+        for (String file : classpathList) {
+            urlList.add(new File(file).toURI().toURL());
         }
-        ConfigurationBuilder confBuilder = new ConfigurationBuilder();
-        confBuilder.setUrls(urls);
-        return new Reflections(confBuilder);
+        return new Reflections(new ConfigurationBuilder().setUrls(urlList));
     }
 
     private final Reflections reflections;
@@ -70,7 +60,7 @@ public ConnectorDocGenerator() throws Exception {
         this.reflections = newReflections();
     }
 
-    private void generateConnectorYaml(Class configClass, PrintWriter writer) {
+    private void generateConnectorYamlFile(Class configClass, PrintWriter writer) {
         log.info("Processing connector config class : {}", configClass);
 
         writer.println("configs:");
@@ -82,7 +72,9 @@ private void generateConnectorYaml(Class configClass, PrintWriter writer) {
             }
             FieldDoc fieldDoc = field.getDeclaredAnnotation(FieldDoc.class);
             if (null == fieldDoc) {
-                throw new RuntimeException("Missing `FieldDoc` for field '" + field.getName() + "'");
+                final String message = "Missing FieldDoc for field '%s' in class '%s'."
+                        .formatted(field.getName(), configClass.getCanonicalName());
+                throw new RuntimeException(message);
             }
             writer.println(INDENT + "# " + fieldDoc.help());
             String fieldPrefix = "";
@@ -99,28 +91,28 @@ private void generateConnectorYaml(Class configClass, PrintWriter writer) {
         writer.flush();
     }
 
-    private void generateConnectorYaml(Class connectorClass, Connector connectorDef, PrintWriter writer) {
+    private void generateConnectorYamlFile(Class connectorClass, Connector connectorDef, PrintWriter writer) {
         log.info("Processing connector definition : {}", connectorDef);
         writer.println("# " + connectorDef.type() + " connector : " + connectorClass.getName());
         writer.println();
         writer.println("# " + connectorDef.help());
         writer.println();
-        generateConnectorYaml(connectorDef.configClass(), writer);
+        generateConnectorYamlFile(connectorDef.configClass(), writer);
     }
 
-    private void generatorConnectorYamls(String outputDir) throws IOException  {
+    private void generatorConnectorYamlFiles(String outputDir) throws IOException {
         Set> connectorClasses = reflections.getTypesAnnotatedWith(Connector.class);
         log.info("Retrieve all `Connector` annotated classes : {}", connectorClasses);
 
         for (Class connectorClass : connectorClasses) {
-            Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class);
-            try (FileOutputStream fos = new FileOutputStream(
-                    Paths.get(
-                    outputDir,
-                    "pulsar-io-" + connectorDef.name()
-                        + "-" + connectorDef.type().name().toLowerCase()).toString() + ".yml")) {
+            final Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class);
+            final String name = connectorDef.name().toLowerCase();
+            final String type = connectorDef.type().name().toLowerCase();
+            final String filename = "pulsar-io-%s-%s.yml".formatted(name, type);
+            final Path outputPath = Path.of(outputDir, filename);
+            try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) {
                 PrintWriter pw = new PrintWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8));
-                generateConnectorYaml(connectorClass, connectorDef, pw);
+                generateConnectorYamlFile(connectorClass, connectorDef, pw);
                 pw.flush();
             }
         }
@@ -130,23 +122,14 @@ private void generatorConnectorYamls(String outputDir) throws IOException  {
      * Args for stats generator.
      */
     private static class MainArgs {
-
         @Parameter(
-            names = {
-                "-o", "--output-dir"
-            },
-            description = "The output dir to dump connector docs",
-            required = true
-        )
+                names = {"-o", "--output-dir"},
+                description = "The output dir to dump connector docs",
+                required = true)
         String outputDir = null;
 
-        @Parameter(
-            names = {
-                "-h", "--help"
-            },
-            description = "Show this help message")
+        @Parameter(names = {"-h", "--help"}, description = "Show this help message")
         boolean help = false;
-
     }
 
     public static void main(String[] args) throws Exception {
@@ -169,7 +152,7 @@ public static void main(String[] args) throws Exception {
         }
 
         ConnectorDocGenerator docGen = new ConnectorDocGenerator();
-        docGen.generatorConnectorYamls(mainArgs.outputDir);
+        docGen.generatorConnectorYamlFiles(mainArgs.outputDir);
     }
 
 }
diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
index a8c7358bf9415..9f42dbda7be1b 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
@@ -242,6 +242,11 @@ public class ElasticSearchConfig implements Serializable {
     )
     private String primaryFields = "";
 
+    @FieldDoc(
+            required = false,
+            defaultValue = "",
+            help = "The SSL config for elastic search."
+    )
     private ElasticSearchSslConfig ssl = new ElasticSearchSslConfig();
 
     @FieldDoc(

From 93fb4f8eb6bd12872f05d7c5ec6c00b719d53009 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Mon, 17 Apr 2023 11:46:34 +0800
Subject: [PATCH 195/404] [improve][build] Upgrade snakeyaml version to 2.0
 (#20085)

---
 buildtools/pom.xml                            |  2 +-
 .../server/src/assemble/LICENSE.bin.txt       | 32 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        | 24 +++++++-------
 pom.xml                                       |  6 ++--
 pulsar-broker-auth-oidc/pom.xml               | 12 +++++++
 pulsar-functions/runtime/pom.xml              | 14 ++++++++
 .../KubernetesSecretsTokenAuthProvider.java   | 10 +++---
 .../runtime/kubernetes/KubernetesRuntime.java | 11 +++----
 .../kubernetes/KubernetesRuntimeFactory.java  |  2 +-
 ...ubernetesSecretsTokenAuthProviderTest.java |  2 +-
 .../KubernetesRuntimeFactoryTest.java         |  6 ++--
 pulsar-functions/secrets/pom.xml              | 14 ++++++++
 pulsar-io/kafka-connect-adaptor/pom.xml       |  7 ++++
 pulsar-sql/presto-distribution/LICENSE        | 30 ++++++++---------
 src/owasp-dependency-check-suppressions.xml   |  8 -----
 15 files changed, 109 insertions(+), 71 deletions(-)

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index b22abf286f693..04b25cced42ac 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -52,7 +52,7 @@
     4.2.3
     31.0.1-jre
     1.10.12
-    1.32
+    2.0
     3.12.4
     
     
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index c321e54de2e01..50ff83a0ab11f 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -246,21 +246,21 @@ The Apache Software License, Version 2.0
  * JCommander -- com.beust-jcommander-1.82.jar
  * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar
  * Jackson
-     - com.fasterxml.jackson.core-jackson-annotations-2.13.4.jar
-     - com.fasterxml.jackson.core-jackson-core-2.13.4.jar
-     - com.fasterxml.jackson.core-jackson-databind-2.13.4.2.jar
-     - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.4.jar
-     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.4.jar
-     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.4.jar
-     - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.4.jar
-     - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.4.jar
-     - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.13.4.jar
-     - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.13.4.jar
-     - com.fasterxml.jackson.module-jackson-module-parameter-names-2.13.4.jar
+     - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar
+     - com.fasterxml.jackson.core-jackson-core-2.14.2.jar
+     - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar
+     - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar
+     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar
+     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar
+     - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar
+     - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar
+     - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar
+     - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar
+     - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
  * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar
- * Bitbucket -- org.bitbucket.b_c-jose4j-0.7.6.jar
+ * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.3.jar
  * Gson
     - com.google.code.gson-gson-2.8.9.jar
     - io.gsonfire-gson-fire-1.8.5.jar
@@ -402,7 +402,7 @@ The Apache Software License, Version 2.0
     - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar
     - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar
     - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar
- * SnakeYaml -- org.yaml-snakeyaml-1.32.jar
+ * SnakeYaml -- org.yaml-snakeyaml-2.0.jar
  * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
  * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar
@@ -453,9 +453,9 @@ The Apache Software License, Version 2.0
   * Apache Yetus
     - org.apache.yetus-audience-annotations-0.12.0.jar
   * Kubernetes Client
-    - io.kubernetes-client-java-12.0.1.jar
-    - io.kubernetes-client-java-api-12.0.1.jar
-    - io.kubernetes-client-java-proto-12.0.1.jar
+    - io.kubernetes-client-java-18.0.0.jar
+    - io.kubernetes-client-java-api-18.0.0.jar
+    - io.kubernetes-client-java-proto-18.0.0.jar
   * Dropwizard
     - io.dropwizard.metrics-metrics-core-4.1.12.1.jar
     - io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 0ac08caa1c01d..b4355ca8a38fa 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses:
 The Apache Software License, Version 2.0
  * JCommander -- jcommander-1.82.jar
  * Jackson
-     - jackson-annotations-2.13.4.jar
-     - jackson-core-2.13.4.jar
-     - jackson-databind-2.13.4.2.jar
-     - jackson-dataformat-yaml-2.13.4.jar
-     - jackson-jaxrs-base-2.13.4.jar
-     - jackson-jaxrs-json-provider-2.13.4.jar
-     - jackson-module-jaxb-annotations-2.13.4.jar
-     - jackson-module-jsonSchema-2.13.4.jar
-     - jackson-datatype-jdk8-2.13.4.jar
-     - jackson-datatype-jsr310-2.13.4.jar
-     - jackson-module-parameter-names-2.13.4.jar
+     - jackson-annotations-2.14.2.jar
+     - jackson-core-2.14.2.jar
+     - jackson-databind-2.14.2.jar
+     - jackson-dataformat-yaml-2.14.2.jar
+     - jackson-jaxrs-base-2.14.2.jar
+     - jackson-jaxrs-json-provider-2.14.2.jar
+     - jackson-module-jaxb-annotations-2.14.2.jar
+     - jackson-module-jsonSchema-2.14.2.jar
+     - jackson-datatype-jdk8-2.14.2.jar
+     - jackson-datatype-jsr310-2.14.2.jar
+     - jackson-module-parameter-names-2.14.2.jar
  * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar
  * Gson
     - gson-2.8.9.jar
@@ -407,7 +407,7 @@ The Apache Software License, Version 2.0
     - websocket-api-9.4.48.v20220622.jar
     - websocket-client-9.4.48.v20220622.jar
     - websocket-common-9.4.48.v20220622.jar
- * SnakeYaml -- snakeyaml-1.32.jar
+ * SnakeYaml -- snakeyaml-2.0.jar
  * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar
  * Javassist -- javassist-3.25.0-GA.jar
   * Apache Avro
diff --git a/pom.xml b/pom.xml
index 094a35b5ed13a..e18222604e621 100644
--- a/pom.xml
+++ b/pom.xml
@@ -148,7 +148,7 @@ flexible messaging model and an intuitive client API.
     1.69
     1.0.6
     1.0.2.3
-    2.13.4.20221013
+    2.14.2
     0.10.2
     1.6.2
     8.37
@@ -224,7 +224,7 @@ flexible messaging model and an intuitive client API.
     2.3.3
     2.0.2
     5.12.1
-    12.0.1
+    18.0.0
     4.9.3
     
     2.8.0
@@ -236,7 +236,7 @@ flexible messaging model and an intuitive client API.
     4.5.13
     4.4.15
     0.5.11
-    1.32
+    2.0
     1.10.12
     5.3.3
     3.4.3
diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml
index 9ad0363775a13..7bf3226da761b 100644
--- a/pulsar-broker-auth-oidc/pom.xml
+++ b/pulsar-broker-auth-oidc/pom.xml
@@ -83,6 +83,18 @@
           io.prometheus
           simpleclient_httpserver
         
+        
+          bcpkix-jdk18on
+          org.bouncycastle
+        
+        
+          bcutil-jdk18on
+          org.bouncycastle
+        
+        
+          bcprov-jdk18on
+          org.bouncycastle
+        
       
     
 
diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml
index 5558ced4c0749..0bc86b83c1668 100644
--- a/pulsar-functions/runtime/pom.xml
+++ b/pulsar-functions/runtime/pom.xml
@@ -64,6 +64,20 @@
       io.kubernetes
       client-java
       ${kubernetesclient.version}
+      
+        
+          bcpkix-jdk18on
+          org.bouncycastle
+        
+        
+          bcutil-jdk18on
+          org.bouncycastle
+        
+        
+          bcprov-jdk18on
+          org.bouncycastle
+        
+      
     
     
       ${project.groupId}
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java
index 1053e6e170ef1..e0543f28ac160 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java
@@ -205,8 +205,7 @@ public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional {
                     try {
-                        coreClient.readNamespacedSecret(secretName, kubeNamespace,
-                                null, null, null);
+                        coreClient.readNamespacedSecret(secretName, kubeNamespace, null);
 
                     } catch (ApiException e) {
                         // statefulset is gone
@@ -305,12 +304,13 @@ private void upsertSecret(String token, Function.FunctionDetails funcDetails, St
                             .data(buildSecretMap(token));
 
                     try {
-                        coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null);
+                        coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null, null);
                     } catch (ApiException e) {
                         if (e.getCode() == HTTP_CONFLICT) {
                             try {
                                 coreClient
-                                        .replaceNamespacedSecret(secretName, kubeNamespace, v1Secret, null, null, null);
+                                        .replaceNamespacedSecret(secretName, kubeNamespace, v1Secret,
+                                                null, null, null, null);
                                 return Actions.ActionResult.builder().success(true).build();
 
                             } catch (ApiException e1) {
@@ -366,7 +366,7 @@ private String createSecret(String token, Function.FunctionDetails funcDetails)
                             .metadata(new V1ObjectMeta().name(getSecretName(id)))
                             .data(buildSecretMap(token));
                     try {
-                        coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null);
+                        coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null, null);
                     } catch (ApiException e) {
                         // already exists
                         if (e.getCode() == HTTP_CONFLICT) {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
index e206862b68a67..0a5e1268c89a5 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
@@ -472,7 +472,7 @@ private void submitService() throws Exception {
                 .supplier(() -> {
                     final V1Service response;
                     try {
-                        response = coreClient.createNamespacedService(jobNamespace, service, null, null, null);
+                        response = coreClient.createNamespacedService(jobNamespace, service, null, null, null, null);
                     } catch (ApiException e) {
                         // already exists
                         if (e.getCode() == HTTP_CONFLICT) {
@@ -561,7 +561,8 @@ private void submitStatefulSet() throws Exception {
                 .supplier(() -> {
                     final V1StatefulSet response;
                     try {
-                        response = appsClient.createNamespacedStatefulSet(jobNamespace, statefulSet, null, null, null);
+                        response = appsClient.createNamespacedStatefulSet(jobNamespace, statefulSet,
+                                null, null, null, null);
                     } catch (ApiException e) {
                         // already exists
                         if (e.getCode() == HTTP_CONFLICT) {
@@ -657,8 +658,7 @@ public void deleteStatefulSet() throws InterruptedException {
                 .supplier(() -> {
                     V1StatefulSet response;
                     try {
-                        response = appsClient.readNamespacedStatefulSet(statefulSetName, jobNamespace,
-                                null, null, null);
+                        response = appsClient.readNamespacedStatefulSet(statefulSetName, jobNamespace, null);
                     } catch (ApiException e) {
                         // statefulset is gone
                         if (e.getCode() == HTTP_NOT_FOUND) {
@@ -805,8 +805,7 @@ public void deleteService() throws InterruptedException {
                 .supplier(() -> {
                     V1Service response;
                     try {
-                        response = coreClient.readNamespacedService(serviceName, jobNamespace,
-                                null, null, null);
+                        response = coreClient.readNamespacedService(serviceName, jobNamespace, null);
 
                     } catch (ApiException e) {
                         // service is gone
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
index 895304138a530..3e1d40e80dc6e 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
@@ -405,7 +405,7 @@ static void fetchConfigMap(CoreV1Api coreClient, String changeConfigMap,
                                KubernetesRuntimeFactory kubernetesRuntimeFactory) {
         try {
             V1ConfigMap v1ConfigMap =
-                    coreClient.readNamespacedConfigMap(changeConfigMap, changeConfigMapNamespace, null, true, false);
+                    coreClient.readNamespacedConfigMap(changeConfigMap, changeConfigMapNamespace, null);
             Map data = v1ConfigMap.getData();
             if (data != null) {
                 overRideKubernetesConfig(data, kubernetesRuntimeFactory);
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java
index 081e693b6a321..cf294afcf9b78 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java
@@ -103,7 +103,7 @@ public void testConfigureAuthDataStatefulSetNoCa() {
     @Test
     public void testCacheAuthData() throws ApiException {
         CoreV1Api coreV1Api = mock(CoreV1Api.class);
-        doReturn(new V1Secret()).when(coreV1Api).createNamespacedSecret(anyString(), any(), anyString(), anyString(), anyString());
+        doReturn(new V1Secret()).when(coreV1Api).createNamespacedSecret(anyString(), any(), anyString(), anyString(), anyString(), anyString());
         KubernetesSecretsTokenAuthProvider kubernetesSecretsTokenAuthProvider = new KubernetesSecretsTokenAuthProvider();
         kubernetesSecretsTokenAuthProvider.initialize(coreV1Api,  null, (fd) -> "default");
         Function.FunctionDetails funcDetails = Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-ns").setName("test-func").build();
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java
index a5fc8f231a6b5..48497bf218d40 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java
@@ -468,9 +468,9 @@ public void testDynamicConfigMapLoading() throws Exception {
         KubernetesRuntimeFactory kubernetesRuntimeFactory = getKuberentesRuntimeFactory();
         CoreV1Api coreV1Api = Mockito.mock(CoreV1Api.class);
         V1ConfigMap v1ConfigMap = new V1ConfigMap();
-        Mockito.doReturn(v1ConfigMap).when(coreV1Api).readNamespacedConfigMap(any(), any(), any(), any(), any());
+        Mockito.doReturn(v1ConfigMap).when(coreV1Api).readNamespacedConfigMap(any(), any(), any());
         KubernetesRuntimeFactory.fetchConfigMap(coreV1Api, changeConfigMap, changeConfigNamespace, kubernetesRuntimeFactory);
-        Mockito.verify(coreV1Api, Mockito.times(1)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null), eq(true), eq(false));
+        Mockito.verify(coreV1Api, Mockito.times(1)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null));
         KubernetesRuntimeFactory expected = getKuberentesRuntimeFactory();
         assertEquals(kubernetesRuntimeFactory, expected);
 
@@ -479,7 +479,7 @@ public void testDynamicConfigMapLoading() throws Exception {
         configs.put("imagePullPolicy", "test_imagePullPolicy2");
         v1ConfigMap.setData(configs);
         KubernetesRuntimeFactory.fetchConfigMap(coreV1Api, changeConfigMap, changeConfigNamespace, kubernetesRuntimeFactory);
-        Mockito.verify(coreV1Api, Mockito.times(2)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null), eq(true), eq(false));
+        Mockito.verify(coreV1Api, Mockito.times(2)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null));
 
        assertEquals(kubernetesRuntimeFactory.getPulsarDockerImageName(), "test_dockerImage2");
        assertEquals(kubernetesRuntimeFactory.getImagePullPolicy(), "test_imagePullPolicy2");
diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml
index 08d4e9bf63ab8..f1fc43fcdcc5a 100644
--- a/pulsar-functions/secrets/pom.xml
+++ b/pulsar-functions/secrets/pom.xml
@@ -35,6 +35,20 @@
       io.kubernetes
       client-java
       ${kubernetesclient.version}
+      
+        
+          bcpkix-jdk18on
+          org.bouncycastle
+        
+        
+          bcutil-jdk18on
+          org.bouncycastle
+        
+        
+          bcprov-jdk18on
+          org.bouncycastle
+        
+      
     
 
     
diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml
index 12ebda087eb3d..8651b378ed118 100644
--- a/pulsar-io/kafka-connect-adaptor/pom.xml
+++ b/pulsar-io/kafka-connect-adaptor/pom.xml
@@ -166,6 +166,13 @@
       test
     
 
+    
+      org.bouncycastle
+      bc-fips
+      ${bouncycastle.bc-fips.version}
+      test
+    
+
     
       com.typesafe.netty
       netty-reactive-streams
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 214a928a74941..d281ba1f3643e 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses:
 The Apache Software License, Version 2.0
 
   * Jackson
-    - jackson-annotations-2.13.4.jar
-    - jackson-core-2.13.4.jar
-    - jackson-databind-2.13.4.2.jar
-    - jackson-dataformat-smile-2.13.4.jar
-    - jackson-datatype-guava-2.13.4.jar
-    - jackson-datatype-jdk8-2.13.4.jar
-    - jackson-datatype-joda-2.13.4.jar
-    - jackson-datatype-jsr310-2.13.4.jar
-    - jackson-dataformat-yaml-2.13.4.jar
-    - jackson-jaxrs-base-2.13.4.jar
-    - jackson-jaxrs-json-provider-2.13.4.jar
-    - jackson-module-jaxb-annotations-2.13.4.jar
-    - jackson-module-jsonSchema-2.13.4.jar
+    - jackson-annotations-2.14.2.jar
+    - jackson-core-2.14.2.jar
+    - jackson-databind-2.14.2.jar
+    - jackson-dataformat-smile-2.14.2.jar
+    - jackson-datatype-guava-2.14.2.jar
+    - jackson-datatype-jdk8-2.14.2.jar
+    - jackson-datatype-joda-2.14.2.jar
+    - jackson-datatype-jsr310-2.14.2.jar
+    - jackson-dataformat-yaml-2.14.2.jar
+    - jackson-jaxrs-base-2.14.2.jar
+    - jackson-jaxrs-json-provider-2.14.2.jar
+    - jackson-module-jaxb-annotations-2.14.2.jar
+    - jackson-module-jsonSchema-2.14.2.jar
  * Guava
     - guava-31.0.1-jre.jar
     - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
@@ -401,7 +401,7 @@ The Apache Software License, Version 2.0
   * RocksDB JNI
     - rocksdbjni-7.9.2.jar
   * SnakeYAML
-    - snakeyaml-1.32.jar
+    - snakeyaml-2.0.jar
   * Bean Validation API
     - validation-api-2.0.1.Final.jar
   * Objectsize
@@ -456,7 +456,7 @@ The Apache Software License, Version 2.0
   * Snappy
     - snappy-java-1.1.8.4.jar
   * Jackson
-    - jackson-module-parameter-names-2.13.4.jar
+    - jackson-module-parameter-names-2.14.2.jar
   * Java Assist
     - javassist-3.25.0-GA.jar
   * Java Native Access
diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml
index 84ed2a3332cd4..d599f2de3e85a 100644
--- a/src/owasp-dependency-check-suppressions.xml
+++ b/src/owasp-dependency-check-suppressions.xml
@@ -37,14 +37,6 @@
         .*
     
 
-    
-        
-        e80612549feb5c9191c498de628c1aa80693cf0b
-        CVE-2022-1471
-    
-
     
     
         
Date: Mon, 17 Apr 2023 19:32:13 +0800
Subject: [PATCH 196/404] [improve] [broker] Close temporary open ledger in
 BookkeeperBucketSnapshotStorage (#20111)

---
 .../BookkeeperBucketSnapshotStorage.java      | 33 +++++++++++++++----
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index e7d4f9301dd36..9c30ccf1c0b7e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -66,22 +66,41 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet
 
     @Override
     public CompletableFuture getBucketSnapshotMetadata(long bucketId) {
-        return openLedger(bucketId).thenCompose(
-                ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0).
-                        thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())));
+        return openLedger(bucketId).thenCompose(ledgerHandle -> {
+            CompletableFuture snapshotFuture =
+                    getLedgerEntry(ledgerHandle, 0, 0)
+                            .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()));
+
+            snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle));
+
+            return snapshotFuture;
+        });
     }
 
     @Override
     public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId,
                                                                              long lastSegmentEntryId) {
-        return openLedger(bucketId).thenCompose(
-                ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId,
-                        lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries));
+        return openLedger(bucketId).thenCompose(ledgerHandle -> {
+            CompletableFuture> parseFuture =
+                    getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId)
+                            .thenApply(this::parseSnapshotSegmentEntries);
+
+            parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle));
+
+            return parseFuture;
+        });
     }
 
     @Override
     public CompletableFuture getBucketSnapshotLength(long bucketId) {
-        return openLedger(bucketId).thenApply(LedgerHandle::getLength);
+        return openLedger(bucketId).thenCompose(ledgerHandle -> {
+            CompletableFuture lengthFuture =
+                    CompletableFuture.completedFuture(ledgerHandle.getLength());
+
+            lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle));
+
+            return lengthFuture;
+        });
     }
 
     @Override

From 092819b5c4a68dbc39d9a650d5370af9455e805d Mon Sep 17 00:00:00 2001
From: xiaolong ran 
Date: Tue, 18 Apr 2023 09:45:19 +0800
Subject: [PATCH 197/404] [improve] [broker] Fix broker restart logic (#20113)

---
 .../java/org/apache/pulsar/broker/PulsarService.java | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index aad7d32f732c5..5fc9920d0f215 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -467,6 +467,12 @@ public CompletableFuture closeAsync() {
                 protocolHandlers = null;
             }
 
+            // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker
+            if (this.loadSheddingTask != null) {
+                this.loadSheddingTask.cancel();
+            }
+            executorServicesShutdown.shutdown(loadManagerExecutor);
+
             List> asyncCloseFutures = new ArrayList<>();
             if (this.brokerService != null) {
                 CompletableFuture brokerCloseFuture = this.brokerService.closeAsync();
@@ -501,12 +507,6 @@ public CompletableFuture closeAsync() {
                 this.leaderElectionService = null;
             }
 
-            // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker
-            if (this.loadSheddingTask != null) {
-                this.loadSheddingTask.cancel();
-            }
-            executorServicesShutdown.shutdown(loadManagerExecutor);
-
             if (adminClient != null) {
                 adminClient.close();
                 adminClient = null;

From 6cfa4683a44e7cce39fa6cb70e0fe1fb3d5eae56 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Mon, 17 Apr 2023 22:57:48 -0700
Subject: [PATCH 198/404] [improve][broker] Fix
 ServiceUnitStateCompactionStrategy to cover fast-forward cursor behavior
 after compaction (#20110)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement: https://github.com/apache/pulsar/issues/16691

After the compaction, the cursor can fast-forward to the compacted horizon when a large number of messages are compacted before the next read. Hence, ServiceUnitStateCompactionStrategy also needs to cover this case. Currently, the existing and slow(their states are far behind) tableviews with ServiceUnitStateCompactionStrategy could not accept those compacted messages. In the load balance extension context, this means the ownership data could be inconsistent among brokers.

### Modifications
This PR
  - fixes ServiceUnitStateCompactionStrategy to accept the state data if its version is bigger than the current version +1.
  - (minor fix) does not repeatedly update the replication_clusters in the policies when creating the system namespace. This update redundantly triggers ZK watchers when restarting brokers.
  -  sets closeWithoutWaitingClientDisconnect=true, upon unload(following the same setting as the modular LM's)
---
 .../pulsar/PulsarClusterMetadataSetup.java    | 17 +++-
 .../channel/ServiceUnitStateChannelImpl.java  |  2 +-
 .../ServiceUnitStateCompactionStrategy.java   | 10 ++-
 .../StrategicTwoPhaseCompactor.java           |  2 +-
 .../ExtensibleLoadManagerImplTest.java        |  2 +
 ...erviceUnitStateCompactionStrategyTest.java |  4 +
 .../ServiceUnitStateCompactionTest.java       | 79 +++++++++++++++++++
 .../pulsar/client/impl/TableViewImpl.java     |  7 ++
 8 files changed, 114 insertions(+), 9 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
index 0badbda1afdfd..9b757c55ccd1d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
@@ -385,10 +385,19 @@ static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName nam
             namespaceResources.createPolicies(namespaceName, policies);
         } else {
             log.info("Namespace {} already exists.", namespaceName);
-            namespaceResources.setPolicies(namespaceName, policies -> {
-                policies.replication_clusters.add(cluster);
-                return policies;
-            });
+            var replicaClusterFound = false;
+            var policiesOptional = namespaceResources.getPolicies(namespaceName);
+            if (policiesOptional.isPresent() && policiesOptional.get().replication_clusters.contains(cluster)) {
+                replicaClusterFound = true;
+            }
+            if (!replicaClusterFound) {
+                namespaceResources.setPolicies(namespaceName, policies -> {
+                    policies.replication_clusters.add(cluster);
+                    return policies;
+                });
+                log.info("Updated namespace:{} policies. Added the replication cluster:{}",
+                        namespaceName, cluster);
+            }
         }
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 68c6440e68e4b..b4c4e7fd5d42a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -819,7 +819,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit) {
         NamespaceBundle bundle = getNamespaceBundle(serviceUnit);
         return pulsar.getBrokerService().unloadServiceUnit(
                         bundle,
-                        false,
+                        true,
                         pulsar.getConfig().getNamespaceBundleUnloadingTimeoutMs(),
                         TimeUnit.MILLISECONDS)
                 .thenApply(numUnloadedTopics -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java
index ceb3ea3e9cb6c..72b05b5cd62c8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java
@@ -52,9 +52,13 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to
             return false;
         }
 
-        // Skip the compaction case where from = null and to.versionId > 1
-        if (from != null && from.versionId() + 1 != to.versionId()) {
-            return true;
+        if (from != null) {
+            if (from.versionId() == Long.MAX_VALUE && to.versionId() == Long.MIN_VALUE) { // overflow
+            } else if (from.versionId() >= to.versionId()) {
+                return true;
+            } else if (from.versionId() < to.versionId() - 1) { // Compacted
+                return false;
+            } // else from.versionId() == to.versionId() - 1 // continue to check further
         }
 
         if (to.force()) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
index 557d4a6580161..a6b0942742763 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
@@ -379,7 +379,7 @@ private  CompletableFuture runPhaseTwo(
                             });
                 })
                 .thenCompose(v -> {
-                    log.info("Acking ledger id {}", phaseOneResult.firstId);
+                    log.info("Acking ledger id {}", phaseOneResult.lastId);
                     return ((CompactionReaderImpl) reader)
                             .acknowledgeCumulativeAsync(
                                     phaseOneResult.lastId, Map.of(COMPACTED_TOPIC_LEDGER_PROPERTY,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 9ab2467d0ef1b..b71eeb4745b87 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -592,6 +592,8 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except
         restartBroker();
         pulsar1 = pulsar;
         setPrimaryLoadManager();
+        admin.namespaces().setNamespaceReplicationClusters("public/default",
+                Sets.newHashSet(this.conf.getClusterName()));
 
         var serviceUnitStateChannelPrimaryNew =
                 (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java
index 64964826af652..62de91dab292b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java
@@ -85,6 +85,10 @@ public void testVersionId(){
                 new ServiceUnitStateData(Owned, dst, src, 10),
                 new ServiceUnitStateData(Releasing, "broker2", dst, 5)));
 
+        assertFalse(strategy.shouldKeepLeft(
+                new ServiceUnitStateData(Owned, dst, src, 10),
+                new ServiceUnitStateData(Owned, "broker2", dst, 12)));
+
     }
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java
index 1a69a86f7c6b2..e4f0750a981c9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java
@@ -27,6 +27,9 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE;
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
@@ -49,6 +52,7 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.commons.lang.reflect.FieldUtils;
@@ -69,6 +73,7 @@
 import org.apache.pulsar.client.api.Reader;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
+import org.apache.pulsar.client.impl.ReaderImpl;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
 import org.apache.pulsar.common.policies.data.RetentionPolicies;
@@ -628,6 +633,80 @@ public void testSlowTableviewAfterCompaction() throws Exception {
 
     }
 
+    @Test
+    public void testSlowReceiveTableviewAfterCompaction() throws Exception {
+        String topic = "persistent://my-property/use/my-ns/my-topic1";
+        String strategyClassName = "topicCompactionStrategyClassName";
+
+        pulsarClient.newConsumer(schema)
+                .topic(topic)
+                .subscriptionName("sub1")
+                .readCompacted(true)
+                .subscribe().close();
+
+        var tv = pulsar.getClient().newTableViewBuilder(schema)
+                .topic(topic)
+                .subscriptionName("slowTV")
+                .loadConf(Map.of(
+                        strategyClassName,
+                        ServiceUnitStateCompactionStrategy.class.getName()))
+                .create();
+
+        // Configure retention to ensue data is retained for reader
+        admin.namespaces().setRetention("my-property/use/my-ns",
+                new RetentionPolicies(-1, -1));
+
+        Producer producer = pulsarClient.newProducer(schema)
+                .topic(topic)
+                .compressionType(MSG_COMPRESSION_TYPE)
+                .enableBatching(true)
+                .messageRoutingMode(MessageRoutingMode.SinglePartition)
+                .create();
+
+        StrategicTwoPhaseCompactor compactor
+                = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
+
+        var reader = ((CompletableFuture>) FieldUtils
+                .readDeclaredField(tv, "reader", true)).get();
+        var consumer = spy(reader.getConsumer());
+        FieldUtils.writeDeclaredField(reader, "consumer", consumer, true);
+        String bundle = "bundle1";
+        final AtomicInteger versionId = new AtomicInteger(0);
+        final AtomicInteger cnt = new AtomicInteger(1);
+        int msgAddCount = 1000; // has to be big enough to cover compacted cursor fast-forward.
+        doAnswer(invocationOnMock -> {
+            if (cnt.decrementAndGet() == 0) {
+                var msg = consumer.receiveAsync();
+                for (int i = 0; i < msgAddCount; i++) {
+                    producer.newMessage().key(bundle).value(
+                            new ServiceUnitStateData(Owned, "broker" + versionId.incrementAndGet(), true,
+                                    versionId.get())).send();
+                }
+                compactor.compact(topic, strategy).join();
+                return msg;
+            }
+            // Call the real method
+            reset(consumer);
+            return consumer.receiveAsync();
+        }).when(consumer).receiveAsync();
+        producer.newMessage().key(bundle).value(
+                new ServiceUnitStateData(Owned, "broker", true,
+                        versionId.incrementAndGet())).send();
+        producer.newMessage().key(bundle).value(
+                new ServiceUnitStateData(Owned, "broker" + versionId.incrementAndGet(), true,
+                        versionId.get())).send();
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(
+                () -> {
+                    var val = tv.get(bundle);
+                    assertNotNull(val);
+                    assertEquals(val.dstBroker(), "broker" + versionId.get());
+                }
+        );
+
+        producer.close();
+        tv.close();
+    }
+
     @Test
     public void testBrokerRestartAfterCompaction() throws Exception {
         String topic = "persistent://my-property/use/my-ns/my-topic1";
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
index ff5b251ad55ff..77aba7e48cbad 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
@@ -192,6 +192,13 @@ private void handleMessage(Message msg) {
                 if (compactionStrategy != null) {
                     T prev = data.get(key);
                     update = !compactionStrategy.shouldKeepLeft(prev, cur);
+                    if (!update) {
+                        log.info("Skipped the message from topic {}. key={} value={} prev={}",
+                                conf.getTopicName(),
+                                key,
+                                cur,
+                                prev);
+                    }
                 }
 
                 if (update) {

From 6c89467c74bf7e4bc7de06ed5e287643819118ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gabriel=20Mikl=C3=B3s?= 
Date: Tue, 18 Apr 2023 11:13:50 +0200
Subject: [PATCH 199/404] [improve][fn] Upgrade Kotlin version to1.8.20
 (#20114)

---
 distribution/server/src/assemble/LICENSE.bin.txt | 8 ++++----
 pom.xml                                          | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 50ff83a0ab11f..a45c5b7f04c14 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -412,10 +412,10 @@ The Apache Software License, Version 2.0
  * Okio - com.squareup.okio-okio-2.8.0.jar
  * Javassist -- org.javassist-javassist-3.25.0-GA.jar
  * Kotlin Standard Lib
-     - org.jetbrains.kotlin-kotlin-stdlib-1.4.32.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-common-1.4.32.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.4.32.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.4.32.jar
+     - org.jetbrains.kotlin-kotlin-stdlib-1.8.20.jar
+     - org.jetbrains.kotlin-kotlin-stdlib-common-1.8.20.jar
+     - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.8.20.jar
+     - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
      - org.jetbrains-annotations-13.0.jar
  * gRPC
     - io.grpc-grpc-all-1.45.1.jar
diff --git a/pom.xml b/pom.xml
index e18222604e621..0945b00c8d659 100644
--- a/pom.xml
+++ b/pom.xml
@@ -229,7 +229,7 @@ flexible messaging model and an intuitive client API.
     
     2.8.0
     
-    1.4.32
+    1.8.20
     1.0
     9.1.6
     5.3.26

From 78cf5a6097fe4b70e51724921ce174bcac919a77 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Wed, 19 Apr 2023 01:27:39 +0800
Subject: [PATCH 200/404] [fix][admin] Add javax.xml.bind:jaxb-api to shade
 (#20106)

Signed-off-by: nodece 
---
 pulsar-client-admin-shaded/pom.xml | 5 +++++
 pulsar-client-all/pom.xml          | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
index cf669b5869afb..b7c2d996bc5e5 100644
--- a/pulsar-client-admin-shaded/pom.xml
+++ b/pulsar-client-admin-shaded/pom.xml
@@ -131,6 +131,7 @@
                   com.yahoo.datasketches:sketches-core
                   org.glassfish.jersey*:*
                   javax.ws.rs:*
+                  javax.xml.bind:jaxb-api
                   jakarta.annotation:*
                   org.glassfish.hk2*:*
                   io.grpc:*
@@ -230,6 +231,10 @@
                   javax.annotation
                   org.apache.pulsar.shade.javax.annotation
                 
+                
+                  javax.xml.bind
+                  org.apache.pulsar.shade.javax.xml.bind
+                
                 
                   jersey
                   org.apache.pulsar.shade.jersey
diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index 73621954e1fb8..cf39c5db256ae 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -182,6 +182,7 @@
                   com.yahoo.datasketches:sketches-core
                   org.glassfish.jersey*:*
                   javax.ws.rs:*
+                  javax.xml.bind:jaxb-api
                   jakarta.annotation:*
                   org.glassfish.hk2*:*
                   io.grpc:*
@@ -285,6 +286,10 @@
                   javax.annotation
                   org.apache.pulsar.shade.javax.annotation
                 
+                
+                  javax.xml.bind
+                  org.apache.pulsar.shade.javax.xml.bind
+                
                 
                   jersey
                   org.apache.pulsar.shade.jersey

From bdb2a96a41a4ada356888fe01fc1a8e0a345ddd4 Mon Sep 17 00:00:00 2001
From: Hang Chen 
Date: Wed, 19 Apr 2023 03:01:31 +0800
Subject: [PATCH 201/404] [improve] [broker] Upgrade the BookKeeper dependency
 to 4.16.1 (#20127)

---
 .../server/src/assemble/LICENSE.bin.txt       | 56 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        |  6 +-
 pom.xml                                       |  2 +-
 pulsar-sql/presto-distribution/LICENSE        | 30 +++++-----
 4 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index a45c5b7f04c14..26651246ddf48 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -346,34 +346,34 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar
-    - org.apache.bookkeeper-circe-checksum-4.16.0.jar
-    - org.apache.bookkeeper-cpu-affinity-4.16.0.jar
-    - org.apache.bookkeeper-statelib-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-api-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-common-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-server-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar
-    - org.apache.bookkeeper.http-http-server-4.16.0.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar
-    - org.apache.distributedlog-distributedlog-common-4.16.0.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.0.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar
-    - org.apache.bookkeeper-native-io-4.16.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar
+    - org.apache.bookkeeper-circe-checksum-4.16.1.jar
+    - org.apache.bookkeeper-cpu-affinity-4.16.1.jar
+    - org.apache.bookkeeper-statelib-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-api-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-common-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-server-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar
+    - org.apache.bookkeeper.http-http-server-4.16.1.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar
+    - org.apache.distributedlog-distributedlog-common-4.16.1.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.1.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar
+    - org.apache.bookkeeper-native-io-4.16.1.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index b4355ca8a38fa..711890809f1bf 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -390,9 +390,9 @@ The Apache Software License, Version 2.0
     - log4j-web-2.18.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.16.0.jar
-    - cpu-affinity-4.16.0.jar
-    - circe-checksum-4.16.0.jar
+    - bookkeeper-common-allocator-4.16.1.jar
+    - cpu-affinity-4.16.1.jar
+    - circe-checksum-4.16.1.jar
   * AirCompressor
      - aircompressor-0.20.jar
  * AsyncHttpClient
diff --git a/pom.xml b/pom.xml
index 0945b00c8d659..0b5ab09d4581a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API.
     
     1.21
 
-    4.16.0
+    4.16.1
     3.8.1
     1.5.0
     1.10.0
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index d281ba1f3643e..09d1396b70419 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -426,21 +426,21 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
   * Apache Bookkeeper
-    - bookkeeper-common-4.16.0.jar
-    - bookkeeper-common-allocator-4.16.0.jar
-    - bookkeeper-proto-4.16.0.jar
-    - bookkeeper-server-4.16.0.jar
-    - bookkeeper-stats-api-4.16.0.jar
-    - bookkeeper-tools-framework-4.16.0.jar
-    - circe-checksum-4.16.0.jar
-    - codahale-metrics-provider-4.16.0.jar
-    - cpu-affinity-4.16.0.jar
-    - http-server-4.16.0.jar
-    - prometheus-metrics-provider-4.16.0.jar
-    - codahale-metrics-provider-4.16.0.jar
-    - bookkeeper-slogger-api-4.16.0.jar
-    - bookkeeper-slogger-slf4j-4.16.0.jar
-    - native-io-4.16.0.jar
+    - bookkeeper-common-4.16.1.jar
+    - bookkeeper-common-allocator-4.16.1.jar
+    - bookkeeper-proto-4.16.1.jar
+    - bookkeeper-server-4.16.1.jar
+    - bookkeeper-stats-api-4.16.1.jar
+    - bookkeeper-tools-framework-4.16.1.jar
+    - circe-checksum-4.16.1.jar
+    - codahale-metrics-provider-4.16.1.jar
+    - cpu-affinity-4.16.1.jar
+    - http-server-4.16.1.jar
+    - prometheus-metrics-provider-4.16.1.jar
+    - codahale-metrics-provider-4.16.1.jar
+    - bookkeeper-slogger-api-4.16.1.jar
+    - bookkeeper-slogger-slf4j-4.16.1.jar
+    - native-io-4.16.1.jar
   * Apache Commons
     - commons-cli-1.5.0.jar
     - commons-codec-1.15.jar

From 46a65fdc182a34753ebabfa35f63c0f6c765462f Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com>
Date: Tue, 18 Apr 2023 14:37:41 -0700
Subject: [PATCH 202/404] [fix][io] KCA: handle kafka sources that use
 commitRecord (#20121)

---
 .../io/kafka/connect/KafkaConnectSource.java  |  43 ++++++-
 .../connect/ErrRecFileStreamSourceTask.java   |  33 +++++
 .../connect/KafkaConnectSourceErrRecTest.java | 118 ++++++++++++++++++
 3 files changed, 193 insertions(+), 1 deletion(-)
 create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java
 create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java

diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java
index f5f6efd08bd99..f2ee8a8e6cafe 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java
@@ -27,6 +27,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.connect.json.JsonConverterConfig;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.apache.pulsar.client.api.Schema;
@@ -57,6 +59,7 @@ public void open(Map config, SourceContext sourceContext) throws
             config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false);
         }
         log.info("jsonWithEnvelope: {}", jsonWithEnvelope);
+
         super.open(config, sourceContext);
     }
 
@@ -69,17 +72,26 @@ public synchronized KafkaSourceRecord processSourceRecord(final SourceRecord src
 
     private static final AvroData avroData = new AvroData(1000);
 
-    private class KafkaSourceRecord extends AbstractKafkaSourceRecord>
+    public class KafkaSourceRecord extends AbstractKafkaSourceRecord>
             implements KVRecord {
 
+        final int keySize;
+        final int valueSize;
+
+        final SourceRecord srcRecord;
+
         KafkaSourceRecord(SourceRecord srcRecord) {
             super(srcRecord);
+            this.srcRecord = srcRecord;
+
             byte[] keyBytes = keyConverter.fromConnectData(
                     srcRecord.topic(), srcRecord.keySchema(), srcRecord.key());
+            keySize = keyBytes != null ? keyBytes.length : 0;
             this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty();
 
             byte[] valueBytes = valueConverter.fromConnectData(
                     srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value());
+            valueSize = valueBytes != null ? valueBytes.length : 0;
 
             this.value = new KeyValue<>(keyBytes, valueBytes);
 
@@ -145,6 +157,35 @@ public KeyValueEncodingType getKeyValueEncodingType() {
             }
         }
 
+        @Override
+        public void ack() {
+            // first try to commitRecord() for the current record in the batch
+            // then call super.ack() which calls commit() after complete batch of records is processed
+            try {
+                if (log.isDebugEnabled()) {
+                    log.debug("commitRecord() for record: {}", srcRecord);
+                }
+                getSourceTask().commitRecord(srcRecord,
+                        new RecordMetadata(
+                                new TopicPartition(srcRecord.topic() == null
+                                            ? topicName.orElse("UNDEFINED")
+                                            : srcRecord.topic(),
+                                        srcRecord.kafkaPartition() == null ? 0 : srcRecord.kafkaPartition()),
+                                -1L, // baseOffset == -1L means no offset
+                                0, // batchIndex, doesn't matter if baseOffset == -1L
+                                null == srcRecord.timestamp() ? -1L : srcRecord.timestamp(),
+                                keySize, // serializedKeySize
+                                valueSize // serializedValueSize
+                        ));
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                log.error("Source task failed to commit record, "
+                        + "source task should resend data, will get duplicate", e);
+                return;
+            }
+            super.ack();
+        }
+
     }
 
 }
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java
new file mode 100644
index 0000000000000..cbdd4c41bf692
--- /dev/null
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pulsar.io.kafka.connect;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.connect.file.FileStreamSourceTask;
+import org.apache.kafka.connect.source.SourceRecord;
+
+public class ErrRecFileStreamSourceTask extends FileStreamSourceTask {
+
+    @Override
+    public void commitRecord(SourceRecord record, RecordMetadata metadata) throws InterruptedException {
+        throw new org.apache.kafka.connect.errors.ConnectException("blah");
+    }
+
+}
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java
new file mode 100644
index 0000000000000..9872e1fbc7e2f
--- /dev/null
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.kafka.connect;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.connect.file.FileStreamSourceConnector;
+import org.apache.kafka.connect.runtime.TaskConfig;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.common.schema.KeyValue;
+import org.apache.pulsar.functions.api.Record;
+import org.apache.pulsar.io.core.SourceContext;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import java.io.File;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+/**
+ * Test the implementation of {@link KafkaConnectSource}.
+ */
+@Slf4j
+public class KafkaConnectSourceErrRecTest extends ProducerConsumerBase  {
+
+    private Map config = new HashMap<>();
+    private String offsetTopicName;
+    // The topic to publish data to, for kafkaSource
+    private String topicName;
+    private KafkaConnectSource kafkaConnectSource;
+    private File tempFile;
+    private SourceContext context;
+    private PulsarClient client;
+
+    @BeforeMethod
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+
+        config.put(TaskConfig.TASK_CLASS_CONFIG, "org.apache.pulsar.io.kafka.connect.ErrRecFileStreamSourceTask");
+        config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter");
+        config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter");
+
+        this.offsetTopicName = "persistent://my-property/my-ns/kafka-connect-source-offset";
+        config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName);
+
+        this.topicName = "persistent://my-property/my-ns/kafka-connect-source";
+        config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName);
+        tempFile = File.createTempFile("some-file-name", null);
+        config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString());
+        config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE));
+
+        this.context = mock(SourceContext.class);
+        this.client = PulsarClient.builder()
+                .serviceUrl(brokerUrl.toString())
+                .build();
+        when(context.getPulsarClient()).thenReturn(this.client);
+    }
+
+    @AfterMethod(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        if (this.client != null) {
+            this.client.close();
+        }
+        tempFile.delete();
+        super.internalCleanup();
+    }
+
+    @Test
+    public void testCommitRecordCalled() throws Exception {
+        kafkaConnectSource = new KafkaConnectSource();
+        kafkaConnectSource.open(config, context);
+
+        // use FileStreamSourceConnector, each line is a record, need "\n" and end of each record.
+        OutputStream os = Files.newOutputStream(tempFile.toPath());
+
+        String line1 = "This is the first line\n";
+        os.write(line1.getBytes());
+        os.flush();
+        os.close();
+
+        Record> record = kafkaConnectSource.read();
+
+        assertTrue(record instanceof KafkaConnectSource.KafkaSourceRecord);
+
+        try {
+            record.ack();
+            fail("expected exception");
+        } catch (Exception e) {
+            log.info("got exception", e);
+            assertTrue(e instanceof org.apache.kafka.connect.errors.ConnectException);
+        }
+    }
+}

From 58ccf020fc0fa5f7d9b70032ae60c8bf9a3f234f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 18 Apr 2023 20:24:58 -0500
Subject: [PATCH 203/404] [fix][broker] Implement authenticateAsync for
 AuthenticationProviderList (#20132)

PIP: #12105 and #19771

### Motivation

With the implementation of asynchronous authentication in PIP 97, I missed a case in the `AuthenticationProviderList` where we need to implement the `authenticateAsync` methods. This PR is necessary for making the `AuthenticationProviderToken` and the `AuthenticationProviderOpenID` work together, which is necessary for anyone transitioning to `AuthenticationProviderOpenID`.

### Modifications

* Implement `AuthenticationListState#authenticateAsync` using a recursive algorithm that first attempts to authenticate the client using the current `authState` and then tries the remaining options.
* Implement `AuthenticationProviderList#authenticateAsync` using a recursive algorithm that attempts each provider sequentially.
* Add test to `AuthenticationProviderListTest` that exercises this method. It didn't technically fail previously, but it's worth adding.
* Add test to `AuthenticationProviderOpenIDIntegrationTest` to cover the exact failures that were causing problems.
---
 ...ticationProviderOpenIDIntegrationTest.java | 57 +++++++++++
 .../AuthenticationProviderList.java           | 95 ++++++++++++++++++-
 .../AuthenticationProviderListTest.java       | 24 +++++
 3 files changed, 175 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
index 298492652c0a8..0075d70f599d3 100644
--- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
+++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
@@ -31,6 +31,7 @@
 import com.github.tomakehurst.wiremock.WireMockServer;
 import io.jsonwebtoken.SignatureAlgorithm;
 import io.jsonwebtoken.impl.DefaultJwtBuilder;
+import io.jsonwebtoken.io.Decoders;
 import io.jsonwebtoken.security.Keys;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -41,13 +42,19 @@
 import java.util.Base64;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.naming.AuthenticationException;
+import lombok.Cleanup;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataCommand;
+import org.apache.pulsar.broker.authentication.AuthenticationProvider;
+import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
+import org.apache.pulsar.broker.authentication.AuthenticationService;
 import org.apache.pulsar.broker.authentication.AuthenticationState;
+import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils;
 import org.apache.pulsar.common.api.AuthData;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -438,6 +445,56 @@ public void testAuthenticationStateOpenIDForTokenExpiration() throws Exception {
         assertTrue(state.isExpired());
     }
 
+    /**
+     * This test covers the migration scenario where you have both the Token and OpenID providers. It ensures
+     * both kinds of authentication work.
+     * @throws Exception
+     */
+    @Test
+    public void testAuthenticationProviderListStateSuccess() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName(),
+                AuthenticationProviderToken.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer);
+
+        // Set up static token
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        // Use public key for validation
+        String publicKeyStr = AuthTokenUtils.encodeKeyBase64(keyPair.getPublic());
+        props.setProperty("tokenPublicKey", publicKeyStr);
+        // Use private key to generate token
+        String privateKeyStr = AuthTokenUtils.encodeKeyBase64(keyPair.getPrivate());
+        PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr),
+                SignatureAlgorithm.RS256);
+        String staticToken = AuthTokenUtils.createToken(privateKey, "superuser", Optional.empty());
+
+        @Cleanup
+        AuthenticationService service = new AuthenticationService(conf);
+        AuthenticationProvider provider = service.getAuthenticationProvider("token");
+
+        // First, authenticate using OIDC
+        String role = "superuser";
+        String oidcToken = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(oidcToken)).get());
+
+        // Authenticate using the static token
+        assertEquals("superuser", provider.authenticateAsync(new AuthenticationDataCommand(staticToken)).get());
+
+        // Use authenticationState to authentication using OIDC
+        AuthenticationState state1 = service.getAuthenticationProvider("token").newAuthState(null, null, null);
+        assertNull(state1.authenticateAsync(AuthData.of(oidcToken.getBytes())).get());
+        assertEquals(state1.getAuthRole(), role);
+
+        // Use authenticationState to authentication using static token
+        AuthenticationState state2 = service.getAuthenticationProvider("token").newAuthState(null, null, null);
+        assertNull(state2.authenticateAsync(AuthData.of(staticToken.getBytes())).get());
+        assertEquals(state1.getAuthRole(), role);
+    }
+
     @Test
     void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception {
         AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java
index f8a96aa624e12..16d6f9859a0b4 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java
@@ -22,6 +22,7 @@
 import java.net.SocketAddress;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import javax.naming.AuthenticationException;
 import javax.net.ssl.SSLSession;
 import javax.servlet.http.HttpServletRequest;
@@ -76,9 +77,12 @@ static  T applyAuthProcessor(List processors, AuthProcessor authF
     private static class AuthenticationListState implements AuthenticationState {
 
         private final List states;
-        private AuthenticationState authState;
+        private volatile AuthenticationState authState;
 
         AuthenticationListState(List states) {
+            if (states == null || states.isEmpty()) {
+                throw new IllegalArgumentException("Authentication state requires at least one state");
+            }
             this.states = states;
             this.authState = states.get(0);
         }
@@ -96,6 +100,61 @@ public String getAuthRole() throws AuthenticationException {
             return getAuthState().getAuthRole();
         }
 
+        @Override
+        public CompletableFuture authenticateAsync(AuthData authData) {
+            // First, attempt to authenticate with the current auth state
+            CompletableFuture authChallengeFuture = new CompletableFuture<>();
+            authState
+                    .authenticateAsync(authData)
+                    .whenComplete((authChallenge, ex) -> {
+                        if (ex == null) {
+                            // Current authState is still correct. Just need to return the authChallenge.
+                            authChallengeFuture.complete(authChallenge);
+                        } else {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Authentication failed for auth provider " + authState.getClass() + ": ", ex);
+                            }
+                            authenticateRemainingAuthStates(authChallengeFuture, authData, ex, states.size() - 1);
+                        }
+                    });
+            return authChallengeFuture;
+        }
+
+        private void authenticateRemainingAuthStates(CompletableFuture authChallengeFuture,
+                                                     AuthData clientAuthData,
+                                                     Throwable previousException,
+                                                     int index) {
+            if (index < 0) {
+                if (previousException == null) {
+                    previousException = new AuthenticationException("Authentication required");
+                }
+                AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(),
+                        "authentication-provider-list", "Authentication required");
+                authChallengeFuture.completeExceptionally(previousException);
+                return;
+            }
+            AuthenticationState state = states.get(index);
+            if (state == authState) {
+                // Skip the current auth state
+                authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, null, index - 1);
+            } else {
+                state.authenticateAsync(clientAuthData)
+                        .whenComplete((authChallenge, ex) -> {
+                            if (ex == null) {
+                                // Found the correct auth state
+                                authState = state;
+                                authChallengeFuture.complete(authChallenge);
+                            } else {
+                                if (log.isDebugEnabled()) {
+                                    log.debug("Authentication failed for auth provider "
+                                            + authState.getClass() + ": ", ex);
+                                }
+                                authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, ex, index - 1);
+                            }
+                        });
+            }
+        }
+
         @Override
         public AuthData authenticate(AuthData authData) throws AuthenticationException {
             return applyAuthProcessor(
@@ -160,6 +219,40 @@ public String getAuthMethodName() {
         return providers.get(0).getAuthMethodName();
     }
 
+    @Override
+    public CompletableFuture authenticateAsync(AuthenticationDataSource authData) {
+        CompletableFuture roleFuture = new CompletableFuture<>();
+        authenticateRemainingAuthProviders(roleFuture, authData, null, providers.size() - 1);
+        return roleFuture;
+    }
+
+    private void authenticateRemainingAuthProviders(CompletableFuture roleFuture,
+                                                    AuthenticationDataSource authData,
+                                                    Throwable previousException,
+                                                    int index) {
+        if (index < 0) {
+            if (previousException == null) {
+                previousException = new AuthenticationException("Authentication required");
+            }
+            AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(),
+                    "authentication-provider-list", "Authentication required");
+            roleFuture.completeExceptionally(previousException);
+            return;
+        }
+        AuthenticationProvider provider = providers.get(index);
+        provider.authenticateAsync(authData)
+                .whenComplete((role, ex) -> {
+                    if (ex == null) {
+                        roleFuture.complete(role);
+                    } else {
+                        if (log.isDebugEnabled()) {
+                            log.debug("Authentication failed for auth provider " + provider.getClass() + ": ", ex);
+                        }
+                        authenticateRemainingAuthProviders(roleFuture, authData, ex, index - 1);
+                    }
+                });
+        }
+
     @Override
     public String authenticate(AuthenticationDataSource authData) throws AuthenticationException {
         return applyAuthProcessor(
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java
index df011412fee85..7793a5c029f2a 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java
@@ -161,6 +161,30 @@ public void testAuthenticate() throws Exception {
         testAuthenticate(tokenBB, SUBJECT_B);
     }
 
+    private void testAuthenticateAsync(String token, String expectedSubject) throws Exception {
+        String actualSubject = authProvider.authenticateAsync(new AuthenticationDataSource() {
+            @Override
+            public boolean hasDataFromCommand() {
+                return true;
+            }
+
+            @Override
+            public String getCommandData() {
+                return token;
+            }
+        }).get();
+        assertEquals(actualSubject, expectedSubject);
+    }
+
+    @Test
+    public void testAuthenticateAsync() throws Exception {
+        testAuthenticateAsync(tokenAA, SUBJECT_A);
+        testAuthenticateAsync(tokenAB, SUBJECT_B);
+        testAuthenticateAsync(tokenBA, SUBJECT_A);
+        testAuthenticateAsync(tokenBB, SUBJECT_B);
+    }
+
+
     private AuthenticationState newAuthState(String token, String expectedSubject) throws Exception {
         // Must pass the token to the newAuthState for legacy reasons.
         AuthenticationState authState = authProvider.newAuthState(

From 14f4b093cf40213b5a8f79bca7a5d6936bc981e0 Mon Sep 17 00:00:00 2001
From: Penghui Li 
Date: Wed, 19 Apr 2023 12:31:25 +0800
Subject: [PATCH 204/404] [fix][broker] Revert : Forbid uploading BYTES schema
 with admin API (#18995) (#20123)

---
 .../admin/impl/SchemasResourceBase.java       |  4 ----
 .../broker/admin/AdminApiSchemaTest.java      | 11 ----------
 .../org/apache/pulsar/schema/SchemaTest.java  | 21 +++++++------------
 3 files changed, 8 insertions(+), 28 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java
index 91a8140fde12b..0bab772044a6d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java
@@ -114,10 +114,6 @@ public CompletableFuture deleteSchemaAsync(boolean authoritative,
     }
 
     public CompletableFuture postSchemaAsync(PostSchemaPayload payload, boolean authoritative) {
-        if (SchemaType.BYTES.name().equals(payload.getType())) {
-            return CompletableFuture.failedFuture(new RestException(Response.Status.NOT_ACCEPTABLE,
-                    "Do not upload a BYTES schema, because it's the default schema type"));
-        }
         return validateOwnershipAndOperationAsync(authoritative, TopicOperation.PRODUCE)
                 .thenCompose(__ -> getSchemaCompatibilityStrategyAsyncWithoutAuth())
                 .thenCompose(schemaCompatibilityStrategy -> {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java
index e8e582f80b0dc..b37114f180216 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java
@@ -185,17 +185,6 @@ private  void testSchemaInfoApi(Schema schema,
 
     }
 
-    @Test
-    public void testCreateBytesSchema() {
-        // forbid admin api creating BYTES schema to be consistent with client side
-        try {
-            testSchemaInfoApi(Schema.BYTES, "schematest/test/test-BYTES");
-            fail("should fail");
-        } catch (Exception e) {
-            assertTrue(e.getMessage().contains("Do not upload a BYTES schema"));
-        }
-    }
-
     @Test(dataProvider = "version")
     public void testPostSchemaCompatibilityStrategy(ApiVersion version) throws PulsarAdminException {
         String namespace = format("%s%s%s", "schematest", (ApiVersion.V1.equals(version) ? "/" + cluster + "/" : "/"),
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
index 0ec72b2ef470a..7ba4529cdbdf4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
@@ -513,17 +513,13 @@ private void testUseAutoConsumeWithSchemalessTopic(SchemaType schema) throws Exc
         admin.topics().createPartitionedTopic(topic, 2);
 
         // set schema
-        if (!schema.equals(SchemaType.BYTES)) {
-            // don't upload bytes schema with admin API
-            // because null schema means the BYTES schema
-            SchemaInfo schemaInfo = SchemaInfoImpl
-                    .builder()
-                    .schema(new byte[0])
-                    .name("dummySchema")
-                    .type(schema)
-                    .build();
-            admin.schemas().createSchema(topic, schemaInfo);
-        }
+        SchemaInfo schemaInfo = SchemaInfoImpl
+                .builder()
+                .schema(new byte[0])
+                .name("dummySchema")
+                .type(schema)
+                .build();
+        admin.schemas().createSchema(topic, schemaInfo);
 
         Producer producer = pulsarClient
                 .newProducer()
@@ -548,8 +544,7 @@ private void testUseAutoConsumeWithSchemalessTopic(SchemaType schema) throws Exc
         Message message2 = consumer2.receive();
         if (schema == SchemaType.BYTES) {
             assertEquals(schema, message.getReaderSchema().get().getSchemaInfo().getType());
-            // default schema of AUTO_CONSUME is BYTES
-            assertTrue(message2.getReaderSchema().get().toString().contains("BYTES"));
+            assertEquals(schema, message2.getReaderSchema().get().getSchemaInfo().getType());
         } else if (schema == SchemaType.NONE) {
             // schema NONE is always reported as BYTES
             assertEquals(SchemaType.BYTES, message.getReaderSchema().get().getSchemaInfo().getType());

From 866d405726bb6527db954201f2260620181747dc Mon Sep 17 00:00:00 2001
From: tison 
Date: Wed, 19 Apr 2023 14:45:29 +0800
Subject: [PATCH 205/404] [fix][sec] spring.version=5.3.27 to fix
 CVE-2023-20863 (#20124)

Signed-off-by: tison 
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 0b5ab09d4581a..22589a1719812 100644
--- a/pom.xml
+++ b/pom.xml
@@ -232,7 +232,7 @@ flexible messaging model and an intuitive client API.
     1.8.20
     1.0
     9.1.6
-    5.3.26
+    5.3.27
     4.5.13
     4.4.15
     0.5.11

From 9b723022436cc1a150af765103e8d343679f92ce Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 19 Apr 2023 16:44:12 +0800
Subject: [PATCH 206/404] [improve][broker] Make timer execute immediately
 after load index (#20126)

---
 .../delayed/bucket/BucketDelayedDeliveryTracker.java      | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 6678c6df254b0..ad2fc6fae4c87 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -541,7 +541,7 @@ public long getBufferMemoryUsage() {
 
     @Override
     public synchronized NavigableSet getScheduledMessages(int maxMessages) {
-        if (!checkPendingOpDone()) {
+        if (!checkPendingLoadDone()) {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.",
                         dispatcher.getName());
@@ -628,11 +628,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                         if (timeout != null) {
                             timeout.cancel();
                         }
-                        timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS);
+                        timeout = timer.newTimeout(this, 0, TimeUnit.MILLISECONDS);
                     }
                 });
 
-                if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) {
+                if (!checkPendingLoadDone() || loadFuture.isCompletedExceptionally()) {
                     break;
                 }
             }
@@ -651,7 +651,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
         return positions;
     }
 
-    private synchronized boolean checkPendingOpDone() {
+    private synchronized boolean checkPendingLoadDone() {
         if (pendingLoad == null || pendingLoad.isDone()) {
             pendingLoad = null;
             return true;

From fc17c1d98a3c1edd975c131d174a9ef69887d9cd Mon Sep 17 00:00:00 2001
From: Shen Liu 
Date: Wed, 19 Apr 2023 23:40:28 +0800
Subject: [PATCH 207/404] [fix][broker] Fix tenant admin authorization bug.
 (#20068)

Co-authored-by: druidliu 
---
 .../authorization/AuthorizationService.java   | 27 +++++--------------
 .../pulsar/broker/auth/AuthorizationTest.java | 22 ++++++++-------
 .../AuthorizationProducerConsumerTest.java    | 16 +++++++++++
 .../proxy/ProxyAuthorizationTest.java         | 14 +++++++---
 4 files changed, 46 insertions(+), 33 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index 0c61219b57a50..a9225f5e48fa9 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -28,6 +28,7 @@
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
 import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.common.naming.NamespaceName;
@@ -182,13 +183,7 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro
         if (!this.conf.isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(true);
         }
-        return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> {
-            if (isSuperUser) {
-                return CompletableFuture.completedFuture(true);
-            } else {
-                return provider.canProduceAsync(topicName, role, authenticationData);
-            }
-        });
+        return provider.allowTopicOperationAsync(topicName, role, TopicOperation.PRODUCE, authenticationData);
     }
 
     /**
@@ -207,13 +202,9 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro
         if (!this.conf.isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(true);
         }
-        return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> {
-            if (isSuperUser) {
-                return CompletableFuture.completedFuture(true);
-            } else {
-                return provider.canConsumeAsync(topicName, role, authenticationData, subscription);
-            }
-        });
+
+        return provider.allowTopicOperationAsync(topicName, role, TopicOperation.CONSUME,
+                new AuthenticationDataSubscription(authenticationData, subscription));
     }
 
     public boolean canProduce(TopicName topicName, String role, AuthenticationDataSource authenticationData)
@@ -289,13 +280,7 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol
         if (!this.conf.isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(true);
         }
-        return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> {
-            if (isSuperUser) {
-                return CompletableFuture.completedFuture(true);
-            } else {
-                return provider.canLookupAsync(topicName, role, authenticationData);
-            }
-        });
+        return provider.allowTopicOperationAsync(topicName, role, TopicOperation.LOOKUP, authenticationData);
     }
 
     public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
index 58cf4ee418ea4..4fce7c50e1c44 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
@@ -79,10 +79,13 @@ public void cleanup() throws Exception {
     public void simple() throws Exception {
         AuthorizationService auth = pulsar.getBrokerService().getAuthorizationService();
 
-        assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
-
+        try {
+            assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
+            fail("Should throw exception when tenant not exist");
+        } catch (Exception ignored) {}
         admin.clusters().createCluster("c1", ClusterData.builder().build());
-        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1")));
+        String tenantAdmin = "role1";
+        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1")));
         waitForChange();
         admin.namespaces().createNamespace("p1/c1/ns1");
         waitForChange();
@@ -215,21 +218,22 @@ public void simple() throws Exception {
                 SubscriptionAuthMode.Prefix);
         waitForChange();
 
-        assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null));
+        assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null));
         assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null));
-        try {
-            assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "sub1"));
-            fail();
-        } catch (Exception ignored) {}
+        // tenant admin can consume all topics, even if SubscriptionAuthMode.Prefix mode
+        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1"));
         try {
             assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "sub2"));
             fail();
         } catch (Exception ignored) {}
 
-        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "role1-sub1"));
+        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "role1-sub1"));
         assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2"));
         assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1"));
 
+        // tenant admin can produce all topics
+        assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null));
+
         admin.namespaces().deleteNamespace("p1/c1/ns1");
         admin.tenants().deleteTenant("p1");
         admin.clusters().deleteCluster("c1");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
index 0ce3b7df07d1f..dd351286d2e5e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
@@ -1035,6 +1035,22 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol
             return CompletableFuture.completedFuture(grantRoles.contains(role));
         }
 
+        @Override
+        public CompletableFuture allowTopicOperationAsync(TopicName topic, String role,
+                                                                   TopicOperation operation,
+                                                                   AuthenticationDataSource authData) {
+            switch (operation) {
+
+                case PRODUCE:
+                    return canProduceAsync(topic, role, authData);
+                case CONSUME:
+                    return canConsumeAsync(topic, role, authData, authData.getSubscription());
+                case LOOKUP:
+                    return canLookupAsync(topic, role, authData);
+            }
+            return super.allowTopicOperationAsync(topic, role, operation, authData);
+        }
+
         @Override
         public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions,
                 String role, String authData) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java
index a3b26a4a9d122..29327d3d4ebe1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java
@@ -25,6 +25,7 @@
 import static org.mockito.Mockito.doReturn;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
 import com.google.common.collect.Sets;
 import java.util.EnumSet;
 import java.util.Optional;
@@ -83,10 +84,13 @@ protected void cleanup() throws Exception {
     public void test() throws Exception {
         AuthorizationService auth = service.getAuthorizationService();
 
-        assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
-
+        try {
+            assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
+            fail("Should throw exception when tenant not exist");
+        } catch (Exception ignored) {}
         admin.clusters().createCluster(configClusterName, ClusterData.builder().build());
-        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1")));
+        String tenantAdmin = "role1";
+        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1")));
         waitForChange();
         admin.namespaces().createNamespace("p1/c1/ns1");
         waitForChange();
@@ -117,6 +121,10 @@ public void test() throws Exception {
         assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
         assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null));
 
+        // tenant admin can produce/consume all topics, even if SubscriptionAuthMode.Prefix mode
+        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1"));
+        assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null));
+
         admin.namespaces().deleteNamespace("p1/c1/ns1");
         admin.tenants().deleteTenant("p1");
         admin.clusters().deleteCluster("c1");

From c9c99aae0d0ca5f8ac20a326d3d76bbf8602c79c Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 19 Apr 2023 10:44:49 -0500
Subject: [PATCH 208/404] [improve] AuthenticationProviderOpenID k8s error logs
 (#20135)

### Motivation

The `AuthenticationProviderOpenID` error logs from the Kubernetes client are not very helpful in certain cases because we only get the error's message and not the error's response body. See https://github.com/kubernetes-client/java/issues/2066 for details on the solution.

Here is an example of a problematic error:

```
org.apache.pulsar.broker.authentication.AuthenticationProviderList - Authentication failed for auth provider class org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID:
javax.naming.AuthenticationException: Error retrieving OpenID Provider Metadata from Kubernetes API server:
	at org.apache.pulsar.broker.authentication.oidc.OpenIDProviderMetadataCache$1.onFailure(OpenIDProviderMetadataCache.java:174) ~[org.apache.pulsar-pulsar-broker-auth-oidc-3.0.0.jar:3.0.0]
	at io.kubernetes.client.openapi.ApiClient$1.onResponse(ApiClient.java:927) ~[io.kubernetes-client-java-api-17.0.2.jar:?]
	at okhttp3.internal.connection.RealCall$AsyncCall.run(RealCall.kt:519) ~[com.squareup.okhttp3-okhttp-4.9.3.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:833) ~[?:?]
```

When I enable debug logging out of the API Client, I can see:

```
INFO: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"forbidden: User \"system:serviceaccount:michael-test:superuser\" cannot get path \"/.well-known/openid-configuration/\"","reason":"Forbidden","details":{},"code":403}

Apr 19, 2023 2:50:25 AM okhttp3.internal.platform.Platform log
INFO: <-- END HTTP (246-byte body)
2023-04-19T02:50:25,832+0000 [pulsar-web-40-1] DEBUG
```

(Note: the solution to this problem is to update the `system:service-account-issuer-discovery` `ClusterRole` to include endpoints with trailing slashes. I created https://github.com/kubernetes/kubernetes/issues/117455 to help solve the permission problem in kubernetes.)

### Modifications

* Use both the message and the response body when converting a Kubernetes client error into a Pulsar Authentication error.

### Verifying this change

This change is a trivial rework / code cleanup without any test coverage.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: no need for a forked PR
---
 .../apache/pulsar/broker/authentication/oidc/JwksCache.java  | 5 +++--
 .../authentication/oidc/OpenIDProviderMetadataCache.java     | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
index 12ea7ec6b906d..b5e038342c2e9 100644
--- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
@@ -130,9 +130,10 @@ private CompletableFuture> getJwksFromKubernetesApiServer() {
                 @Override
                 public void onFailure(ApiException e, int statusCode, Map> responseHeaders) {
                     incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
+                    // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066.
                     future.completeExceptionally(
-                            new AuthenticationException("Failed to retrieve public key from Kubernetes API server: "
-                                    + e.getMessage()));
+                            new AuthenticationException("Failed to retrieve public key from Kubernetes API server. "
+                                    + "Message: " + e.getMessage() + " Response body: " + e.getResponseBody()));
                 }
 
                 @Override
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java
index 33d11f35a349a..111399adbd72b 100644
--- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java
@@ -165,9 +165,10 @@ private CompletableFuture loadOpenIDProviderMetadataForK
                 @Override
                 public void onFailure(ApiException e, int statusCode, Map> responseHeaders) {
                     incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA);
+                    // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066.
                     future.completeExceptionally(new AuthenticationException(
-                            "Error retrieving OpenID Provider Metadata from Kubernetes API server: "
-                                    + e.getMessage()));
+                            "Error retrieving OpenID Provider Metadata from Kubernetes API server. Message: "
+                                    + e.getMessage() + " Response body: " + e.getResponseBody()));
                 }
 
                 @Override

From 4f2b8e21ff5175f09e9e20ca3ad341db59822c86 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 19 Apr 2023 17:42:23 -0500
Subject: [PATCH 209/404] [fix][fn] Supply download auth params when provided
 for k8s runtime (#20144)

PIP: #19849

### Motivation

The new `KubernetesServiceAccountTokenAuthProvider` introduced by #19888  does not configure the authentication for download because there is an unnecessary check that the `getFunctionAuthenticationSpec` is not `null`. Given that we do not use this spec in creating the auth, it is unnecessary to use here. Further, when we construct the function's authentication, we do not have this null check. See:

https://github.com/apache/pulsar/blob/ec102fb024a6ea2b195826778300f20e330dff06/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java#L411-L417

### Modifications

* Update the `KubernetesRuntime` to add authentication params to the download command when provided.

### Verifying this change

A new test is added.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: Skipping for this trivial PR
---
 .../runtime/kubernetes/KubernetesRuntime.java |  3 +--
 .../kubernetes/KubernetesRuntimeTest.java     | 26 +++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
index 0a5e1268c89a5..aa474aad801dc 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
@@ -878,8 +878,7 @@ private List getDownloadCommand(String tenant, String namespace, String
         // add auth plugin and parameters if necessary
         if (authenticationEnabled && authConfig != null) {
             if (isNotBlank(authConfig.getClientAuthenticationPlugin())
-                    && isNotBlank(authConfig.getClientAuthenticationParameters())
-                    && instanceConfig.getFunctionAuthenticationSpec() != null) {
+                    && isNotBlank(authConfig.getClientAuthenticationParameters())) {
                 cmd.addAll(Arrays.asList(
                         "--auth-plugin",
                         authConfig.getClientAuthenticationPlugin(),
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index 1e8194eed443a..3facd37fc9244 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -861,6 +861,32 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception {
         assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand);
     }
 
+    @Test
+    public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws Exception {
+        InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false);
+        config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false));
+
+        factory = createKubernetesRuntimeFactory(null,
+                10, 1.0, 1.0, Optional.empty(), null, wconfig -> {
+                    wconfig.setAuthenticationEnabled(true);
+                }, AuthenticationConfig.builder()
+                        .clientAuthenticationPlugin("com.MyAuth")
+                        .clientAuthenticationParameters("{\"authParam1\": \"authParamValue1\"}")
+                        .build());
+
+        KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l);
+        V1StatefulSet spec = container.createStatefulSet();
+        String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
+                + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
+                + " functions download "
+                + "--tenant " + TEST_TENANT
+                + " --namespace " + TEST_NAMESPACE
+                + " --name " + TEST_NAME
+                + " --destination-file " + pulsarRootDir + "/" + userJarFile;
+        String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2);
+        assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand);
+    }
+
     InstanceConfig createGolangInstanceConfig() {
         InstanceConfig config = new InstanceConfig();
 

From 6dc0b0ea38ab9dc410a24b19fd7567b9e013837d Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Thu, 20 Apr 2023 08:21:08 +0800
Subject: [PATCH 210/404] [improve][broker] Optimize delayed metadata index
 bitmap (#20136)

---
 .../bucket/BucketDelayedDeliveryTracker.java  |  3 ++
 .../delayed/bucket/ImmutableBucket.java       | 53 +++++++++++--------
 .../broker/delayed/bucket/MutableBucket.java  | 11 ++--
 3 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index ad2fc6fae4c87..5d1dc2e27033a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -485,6 +485,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List> asyncLoadNextBucketSnapshotEntry(b
                         this.setLastSegmentEntryId(metadataList.size());
                         this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList);
                         List firstScheduleTimestamps = metadataList.stream().map(
-                                        SnapshotSegmentMetadata::getMinScheduleTimestamp).toList();
+                                SnapshotSegmentMetadata::getMinScheduleTimestamp).toList();
                         this.setFirstScheduleTimestamps(firstScheduleTimestamps);
 
                         return nextSnapshotEntryIndex + 1;
@@ -139,25 +138,37 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
         });
     }
 
+    /**
+     * Recover delayed index bit map and message numbers.
+     * @throws InvalidRoaringFormat invalid bitmap serialization format
+     */
     private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex,
-                                                    List segmentMetadata) {
-        this.delayedIndexBitMap.clear();
-        MutableLong numberMessages = new MutableLong(0);
-        for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) {
-            Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap();
-            bitByteStringMap.forEach((leaderId, bitSetString) -> {
-                boolean exist = this.delayedIndexBitMap.containsKey(leaderId);
-                RoaringBitmap bitSet =
-                        new ImmutableRoaringBitmap(bitSetString.asReadOnlyByteBuffer()).toRoaringBitmap();
-                numberMessages.add(bitSet.getCardinality());
-                if (!exist) {
-                    this.delayedIndexBitMap.put(leaderId, bitSet);
-                } else {
-                    this.delayedIndexBitMap.get(leaderId).or(bitSet);
+                                                    List segmentMetaList) {
+        delayedIndexBitMap.clear(); // cleanup dirty bm
+        final var numberMessages = new MutableLong(0);
+        for (int i = startSnapshotIndex; i < segmentMetaList.size(); i++) {
+            for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) {
+                final var ledgerId = entry.getKey();
+                final var bs = entry.getValue();
+                final var sbm = new RoaringBitmap();
+                try {
+                    sbm.deserialize(bs.asReadOnlyByteBuffer());
+                } catch (IOException e) {
+                    throw new InvalidRoaringFormat(e.getMessage());
                 }
-            });
+                numberMessages.add(sbm.getCardinality());
+                delayedIndexBitMap.compute(ledgerId, (lId, bm) -> {
+                    if (bm == null) {
+                        return sbm;
+                    }
+                    bm.or(sbm);
+                    return bm;
+                });
+            }
         }
-        this.setNumberBucketDelayedMessages(numberMessages.getValue());
+        // optimize bm
+        delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize);
+        setNumberBucketDelayedMessages(numberMessages.getValue());
     }
 
     CompletableFuture> getRemainSnapshotSegment() {
@@ -193,7 +204,7 @@ CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats
                         stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete);
                     } else {
                         log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
-                                 dispatcherName, bucketId, bucketKey);
+                                dispatcherName, bucketId, bucketKey);
 
                         stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete,
                                 System.currentTimeMillis() - deleteStartTime);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index e49ebe9606e01..f404d5d02c15a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -116,10 +116,13 @@ Pair createImmutableBucketAndAsyncPersistent(
 
                 Iterator> iterator = bitMap.entrySet().iterator();
                 while (iterator.hasNext()) {
-                    Map.Entry entry = iterator.next();
-                    byte[] array = new byte[entry.getValue().serializedSizeInBytes()];
-                    entry.getValue().serialize(ByteBuffer.wrap(array));
-                    segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array));
+                    final var entry = iterator.next();
+                    final var lId = entry.getKey();
+                    final var bm = entry.getValue();
+                    bm.runOptimize();
+                    final var array = new byte[bm.serializedSizeInBytes()];
+                    bm.serialize(ByteBuffer.wrap(array));
+                    segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array));
                     iterator.remove();
                 }
 

From d3fa998aa7c0a7a9452079ef2ff05bccf6b273cf Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Thu, 20 Apr 2023 09:26:30 +0800
Subject: [PATCH 211/404] [improve][broker] Cache LedgerHandle in
 BookkeeperBucketSnapshotStorage (#20117)

---
 .../BookkeeperBucketSnapshotStorage.java      | 52 +++++++++----------
 .../BookkeeperBucketSnapshotStorageTest.java  | 43 +++++++++++++++
 2 files changed, 69 insertions(+), 26 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 9c30ccf1c0b7e..18a4c322f7b27 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import javax.validation.constraints.NotNull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.client.BKException;
@@ -48,6 +49,8 @@ public class BookkeeperBucketSnapshotStorage implements BucketSnapshotStorage {
     private final ServiceConfiguration config;
     private BookKeeper bookKeeper;
 
+    private final Map> ledgerHandleFutureCache = new ConcurrentHashMap<>();
+
     public BookkeeperBucketSnapshotStorage(PulsarService pulsar) {
         this.pulsar = pulsar;
         this.config = pulsar.getConfig();
@@ -66,45 +69,30 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet
 
     @Override
     public CompletableFuture getBucketSnapshotMetadata(long bucketId) {
-        return openLedger(bucketId).thenCompose(ledgerHandle -> {
-            CompletableFuture snapshotFuture =
-                    getLedgerEntry(ledgerHandle, 0, 0)
-                            .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()));
-
-            snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle));
-
-            return snapshotFuture;
-        });
+        return getLedgerHandle(bucketId).thenCompose(ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0)
+                .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())));
     }
 
     @Override
     public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId,
                                                                              long lastSegmentEntryId) {
-        return openLedger(bucketId).thenCompose(ledgerHandle -> {
-            CompletableFuture> parseFuture =
-                    getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId)
-                            .thenApply(this::parseSnapshotSegmentEntries);
-
-            parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle));
-
-            return parseFuture;
-        });
+        return getLedgerHandle(bucketId).thenCompose(
+                ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId)
+                        .thenApply(this::parseSnapshotSegmentEntries));
     }
 
     @Override
     public CompletableFuture getBucketSnapshotLength(long bucketId) {
-        return openLedger(bucketId).thenCompose(ledgerHandle -> {
-            CompletableFuture lengthFuture =
-                    CompletableFuture.completedFuture(ledgerHandle.getLength());
-
-            lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle));
-
-            return lengthFuture;
-        });
+        return getLedgerHandle(bucketId).thenCompose(
+                ledgerHandle -> CompletableFuture.completedFuture(ledgerHandle.getLength()));
     }
 
     @Override
     public CompletableFuture deleteBucketSnapshot(long bucketId) {
+        CompletableFuture ledgerHandleFuture = ledgerHandleFutureCache.remove(bucketId);
+        if (ledgerHandleFuture != null) {
+            ledgerHandleFuture.whenComplete((lh, ex) -> closeLedger(lh));
+        }
         return deleteLedger(bucketId);
     }
 
@@ -178,6 +166,18 @@ private CompletableFuture createLedger(String bucketKey, String to
         return future;
     }
 
+    private CompletableFuture getLedgerHandle(Long ledgerId) {
+        CompletableFuture ledgerHandleCompletableFuture =
+                ledgerHandleFutureCache.computeIfAbsent(ledgerId, k -> openLedger(ledgerId));
+        // remove future of completed exceptionally
+        ledgerHandleCompletableFuture.whenComplete((__, ex) -> {
+            if (ex != null) {
+                ledgerHandleFutureCache.remove(ledgerId, ledgerHandleCompletableFuture);
+            }
+        });
+        return ledgerHandleCompletableFuture;
+    }
+
     private CompletableFuture openLedger(Long ledgerId) {
         final CompletableFuture future = new CompletableFuture<>();
         bookKeeper.asyncOpenLedger(
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
index a628b58e10d32..7cb6b8d5865bb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
@@ -30,6 +30,7 @@
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -204,4 +205,46 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted
         Assert.assertTrue(bucketSnapshotLength > 0L);
     }
 
+    @Test
+    public void testConcurrencyGet() throws ExecutionException, InterruptedException {
+        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
+                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+                        .setMinScheduleTimestamp(System.currentTimeMillis())
+                        .setMaxScheduleTimestamp(System.currentTimeMillis())
+                        .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
+
+        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
+                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder()
+                        .addMetadataList(segmentMetadata)
+                        .build();
+        List bucketSnapshotSegments = new ArrayList<>();
+
+        long timeMillis = System.currentTimeMillis();
+        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex =
+                DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L)
+                        .setTimestamp(timeMillis).build();
+        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment =
+                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build();
+        bucketSnapshotSegments.add(snapshotSegment);
+        bucketSnapshotSegments.add(snapshotSegment);
+
+        CompletableFuture future =
+                bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
+        Long bucketId = future.get();
+        Assert.assertNotNull(bucketId);
+
+        List> futures = new ArrayList<>();
+        for (int i = 0; i < 100; i++) {
+            CompletableFuture future0 = CompletableFuture.runAsync(() -> {
+                List list =
+                        bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join();
+                Assert.assertTrue(list.size() > 0);
+            });
+            futures.add(future0);
+        }
+
+        FutureUtil.waitForAll(futures).join();
+    }
+
 }

From dc5e497b2a42910146690c6b4e922e0a80bf1587 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 20 Apr 2023 00:17:02 -0500
Subject: [PATCH 212/404] [fix][broker] Producer/Consumer should call
 allowTopicOperationAsync (#20142)

Fixes: https://github.com/apache/pulsar/issues/20066

### Motivation

In https://github.com/apache/pulsar/pull/20068 we changed the way that the `AuthorizationService` is implemented. I think we should change the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method.

Given that our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, this change helps us move in the right direction.

### Modifications

* Update `Producer` and `Consumer` in broker to call the `AuthorizationService#allowTopicOperationAsync` method.

### Verifying this change

This change is trivial.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: Skipping PR as I ran tests locally.
---
 .../java/org/apache/pulsar/broker/service/Consumer.java   | 8 ++++++--
 .../java/org/apache/pulsar/broker/service/Producer.java   | 3 ++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index 1ee3f513ef288..a3f9da41e6b35 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -43,6 +43,7 @@
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.api.MessageId;
@@ -56,6 +57,7 @@
 import org.apache.pulsar.common.api.proto.MessageIdData;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl;
+import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.schema.SchemaType;
@@ -901,8 +903,10 @@ public String toString() {
     public CompletableFuture checkPermissionsAsync() {
         TopicName topicName = TopicName.get(subscription.getTopicName());
         if (cnx.getBrokerService().getAuthorizationService() != null) {
-            return cnx.getBrokerService().getAuthorizationService().canConsumeAsync(topicName, appId,
-                            cnx.getAuthenticationData(), subscription.getName())
+            AuthenticationDataSubscription authData =
+                    new AuthenticationDataSubscription(cnx.getAuthenticationData(), subscription.getName());
+            return cnx.getBrokerService().getAuthorizationService()
+                    .allowTopicOperationAsync(topicName, TopicOperation.CONSUME, appId, authData)
                     .handle((ok, e) -> {
                         if (e != null) {
                             log.warn("[{}] Get unexpected error while authorizing [{}]  {}", appId,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
index 5b62e3261e64f..53b79f06e8e24 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
@@ -51,6 +51,7 @@
 import org.apache.pulsar.common.api.proto.ServerError;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl;
+import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl;
 import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
 import org.apache.pulsar.common.protocol.Commands;
@@ -781,7 +782,7 @@ public CompletableFuture checkPermissionsAsync() {
         TopicName topicName = TopicName.get(topic.getName());
         if (cnx.getBrokerService().getAuthorizationService() != null) {
             return cnx.getBrokerService().getAuthorizationService()
-                    .canProduceAsync(topicName, appId, cnx.getAuthenticationData())
+                    .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, appId, cnx.getAuthenticationData())
                     .handle((ok, ex) -> {
                         if (ex != null) {
                             log.warn("[{}] Get unexpected error while autorizing [{}]  {}", appId, topic.getName(),

From 00dc7a0691b496065dba6af0c6de64af4973886e Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 20 Apr 2023 00:17:22 -0500
Subject: [PATCH 213/404] [revert] "[fix][broker] Fix tenant admin
 authorization bug. (#20068)" (#20143)

This reverts commit fc17c1d98a3c1edd975c131d174a9ef69887d9cd.

### Motivation

In https://github.com/apache/pulsar/pull/20068 we changed the way that the `AuthorizationService` is implemented. I think this approach could have unintended consequences. Instead, I think we should change the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method. I propose an update to the `Consumer` and `Producer` logic here #20142.

Given that our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, I think we should not change their implementations.

### Modifications

* Revert https://github.com/apache/pulsar/pull/20068

### Verifying this change

This change is trivial. It removes certain test changes that were only made to make the previous PR work.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: Skipping PR as I ran tests locally.
---
 .../authorization/AuthorizationService.java   | 27 ++++++++++++++-----
 .../pulsar/broker/auth/AuthorizationTest.java | 22 +++++++--------
 .../AuthorizationProducerConsumerTest.java    | 16 -----------
 .../proxy/ProxyAuthorizationTest.java         | 14 +++-------
 4 files changed, 33 insertions(+), 46 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index a9225f5e48fa9..0c61219b57a50 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -28,7 +28,6 @@
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
 import org.apache.pulsar.broker.authentication.AuthenticationParameters;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.common.naming.NamespaceName;
@@ -183,7 +182,13 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro
         if (!this.conf.isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(true);
         }
-        return provider.allowTopicOperationAsync(topicName, role, TopicOperation.PRODUCE, authenticationData);
+        return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> {
+            if (isSuperUser) {
+                return CompletableFuture.completedFuture(true);
+            } else {
+                return provider.canProduceAsync(topicName, role, authenticationData);
+            }
+        });
     }
 
     /**
@@ -202,9 +207,13 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro
         if (!this.conf.isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(true);
         }
-
-        return provider.allowTopicOperationAsync(topicName, role, TopicOperation.CONSUME,
-                new AuthenticationDataSubscription(authenticationData, subscription));
+        return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> {
+            if (isSuperUser) {
+                return CompletableFuture.completedFuture(true);
+            } else {
+                return provider.canConsumeAsync(topicName, role, authenticationData, subscription);
+            }
+        });
     }
 
     public boolean canProduce(TopicName topicName, String role, AuthenticationDataSource authenticationData)
@@ -280,7 +289,13 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol
         if (!this.conf.isAuthorizationEnabled()) {
             return CompletableFuture.completedFuture(true);
         }
-        return provider.allowTopicOperationAsync(topicName, role, TopicOperation.LOOKUP, authenticationData);
+        return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> {
+            if (isSuperUser) {
+                return CompletableFuture.completedFuture(true);
+            } else {
+                return provider.canLookupAsync(topicName, role, authenticationData);
+            }
+        });
     }
 
     public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
index 4fce7c50e1c44..58cf4ee418ea4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
@@ -79,13 +79,10 @@ public void cleanup() throws Exception {
     public void simple() throws Exception {
         AuthorizationService auth = pulsar.getBrokerService().getAuthorizationService();
 
-        try {
-            assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
-            fail("Should throw exception when tenant not exist");
-        } catch (Exception ignored) {}
+        assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
+
         admin.clusters().createCluster("c1", ClusterData.builder().build());
-        String tenantAdmin = "role1";
-        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1")));
+        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1")));
         waitForChange();
         admin.namespaces().createNamespace("p1/c1/ns1");
         waitForChange();
@@ -218,22 +215,21 @@ public void simple() throws Exception {
                 SubscriptionAuthMode.Prefix);
         waitForChange();
 
-        assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null));
+        assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null));
         assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null));
-        // tenant admin can consume all topics, even if SubscriptionAuthMode.Prefix mode
-        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1"));
+        try {
+            assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "sub1"));
+            fail();
+        } catch (Exception ignored) {}
         try {
             assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "sub2"));
             fail();
         } catch (Exception ignored) {}
 
-        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "role1-sub1"));
+        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "role1-sub1"));
         assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2"));
         assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1"));
 
-        // tenant admin can produce all topics
-        assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null));
-
         admin.namespaces().deleteNamespace("p1/c1/ns1");
         admin.tenants().deleteTenant("p1");
         admin.clusters().deleteCluster("c1");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
index dd351286d2e5e..0ce3b7df07d1f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
@@ -1035,22 +1035,6 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol
             return CompletableFuture.completedFuture(grantRoles.contains(role));
         }
 
-        @Override
-        public CompletableFuture allowTopicOperationAsync(TopicName topic, String role,
-                                                                   TopicOperation operation,
-                                                                   AuthenticationDataSource authData) {
-            switch (operation) {
-
-                case PRODUCE:
-                    return canProduceAsync(topic, role, authData);
-                case CONSUME:
-                    return canConsumeAsync(topic, role, authData, authData.getSubscription());
-                case LOOKUP:
-                    return canLookupAsync(topic, role, authData);
-            }
-            return super.allowTopicOperationAsync(topic, role, operation, authData);
-        }
-
         @Override
         public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions,
                 String role, String authData) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java
index 29327d3d4ebe1..a3b26a4a9d122 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java
@@ -25,7 +25,6 @@
 import static org.mockito.Mockito.doReturn;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
 import com.google.common.collect.Sets;
 import java.util.EnumSet;
 import java.util.Optional;
@@ -84,13 +83,10 @@ protected void cleanup() throws Exception {
     public void test() throws Exception {
         AuthorizationService auth = service.getAuthorizationService();
 
-        try {
-            assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
-            fail("Should throw exception when tenant not exist");
-        } catch (Exception ignored) {}
+        assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
+
         admin.clusters().createCluster(configClusterName, ClusterData.builder().build());
-        String tenantAdmin = "role1";
-        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1")));
+        admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1")));
         waitForChange();
         admin.namespaces().createNamespace("p1/c1/ns1");
         waitForChange();
@@ -121,10 +117,6 @@ public void test() throws Exception {
         assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null));
         assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null));
 
-        // tenant admin can produce/consume all topics, even if SubscriptionAuthMode.Prefix mode
-        assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1"));
-        assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null));
-
         admin.namespaces().deleteNamespace("p1/c1/ns1");
         admin.tenants().deleteTenant("p1");
         admin.clusters().deleteCluster("c1");

From 575cf2331dd3ac0048923487c6c7904eda4301e6 Mon Sep 17 00:00:00 2001
From: ran 
Date: Thu, 20 Apr 2023 16:18:06 +0800
Subject: [PATCH 214/404] [fix][broker] Fix entry filter feature for the
 non-persistent topic (#20141)

---
 .../nonpersistent/NonPersistentTopic.java     |  5 ++-
 .../service/plugin/FilterEntryTest.java       | 15 +++++--
 .../broker/stats/SubscriptionStatsTest.java   | 42 ++++++++++---------
 3 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index 317b8df6b9a82..33258b06726b5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -26,7 +26,7 @@
 import io.netty.buffer.ByteBuf;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -199,7 +199,8 @@ public void publishMessage(ByteBuf data, PublishContext callback) {
             // entry internally retains data so, duplicateBuffer should be release here
             duplicateBuffer.release();
             if (subscription.getDispatcher() != null) {
-                subscription.getDispatcher().sendMessages(Collections.singletonList(entry));
+                // Dispatcher needs to call the set method to support entry filter feature.
+                subscription.getDispatcher().sendMessages(Arrays.asList(entry));
             } else {
                 // it happens when subscription is created but dispatcher is not created as consumer is not added
                 // yet
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java
index 4b9d91fbde219..b868858646c50 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java
@@ -51,6 +51,7 @@
 import org.apache.pulsar.broker.service.BrokerTestBase;
 import org.apache.pulsar.broker.service.Dispatcher;
 import org.apache.pulsar.broker.service.EntryFilterSupport;
+import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
@@ -286,10 +287,16 @@ public void testFilter() throws Exception {
 
     }
 
+    @DataProvider(name = "topicProvider")
+    public Object[][] topicProvider() {
+        return new Object[][]{
+                {"persistent://prop/ns-abc/topic" + UUID.randomUUID()},
+                {"non-persistent://prop/ns-abc/topic" + UUID.randomUUID()},
+        };
+    }
 
-    @Test
-    public void testFilteredMsgCount() throws Throwable {
-        String topic = "persistent://prop/ns-abc/topic" + UUID.randomUUID();
+    @Test(dataProvider = "topicProvider")
+    public void testFilteredMsgCount(String topic) throws Throwable {
         String subName = "sub";
 
         try (Producer producer = pulsarClient.newProducer(Schema.STRING)
@@ -298,7 +305,7 @@ public void testFilteredMsgCount() throws Throwable {
                      .subscriptionName(subName).subscribe()) {
 
             // mock entry filters
-            PersistentSubscription subscription = (PersistentSubscription) pulsar.getBrokerService()
+            Subscription subscription = pulsar.getBrokerService()
                     .getTopicReference(topic).get().getSubscription(subName);
             Dispatcher dispatcher = subscription.getDispatcher();
             Field field = EntryFilterSupport.class.getDeclaredField("entryFilters");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java
index bf9c1d540bf87..d5e0066a86f15 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java
@@ -211,21 +211,22 @@ public void testSubscriptionStats(final String topic, final String subName, bool
             hasFilterField.set(dispatcher, true);
         }
 
-        for (int i = 0; i < 100; i++) {
-            producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send();
-        }
-        for (int i = 0; i < 100; i++) {
+        int rejectedCount = 100;
+        int acceptCount = 100;
+        int scheduleCount = 100;
+        for (int i = 0; i < rejectedCount; i++) {
             producer.newMessage().property("REJECT", " ").value(UUID.randomUUID().toString()).send();
         }
-        for (int i = 0; i < 100; i++) {
+        for (int i = 0; i < acceptCount; i++) {
+            producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send();
+        }
+        for (int i = 0; i < scheduleCount; i++) {
             producer.newMessage().property("RESCHEDULE", " ").value(UUID.randomUUID().toString()).send();
         }
 
-        for (;;) {
-            Message message = consumer.receive(10, TimeUnit.SECONDS);
-            if (message == null) {
-                break;
-            }
+        for (int i = 0; i < acceptCount; i++) {
+            Message message = consumer.receive(1, TimeUnit.SECONDS);
+            Assert.assertNotNull(message);
             consumer.acknowledge(message);
         }
 
@@ -263,12 +264,12 @@ public void testSubscriptionStats(final String topic, final String subName, bool
                     .mapToDouble(m-> m.value).sum();
 
             if (setFilter) {
-                Assert.assertEquals(filterAccepted, 100);
-                if (isPersistent) {
-                    Assert.assertEquals(filterRejected, 100);
-                    // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount
-                    Assert.assertEquals(throughFilter, filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter);
-                }
+                Assert.assertEquals(filterAccepted, acceptCount);
+                Assert.assertEquals(filterRejected, rejectedCount);
+                // Only works on the test, if there are some markers,
+                // the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount
+                Assert.assertEquals(throughFilter,
+                        filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter);
             } else {
                 Assert.assertEquals(throughFilter, 0D);
                 Assert.assertEquals(filterAccepted, 0D);
@@ -282,19 +283,20 @@ public void testSubscriptionStats(final String topic, final String subName, bool
             Assert.assertEquals(rescheduledMetrics.size(), 0);
         }
 
-        testSubscriptionStatsAdminApi(topic, subName, setFilter);
+        testSubscriptionStatsAdminApi(topic, subName, setFilter, acceptCount, rejectedCount);
     }
 
-    private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter) throws Exception {
+    private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter,
+                                               int acceptCount, int rejectedCount) throws Exception {
         boolean persistent = TopicName.get(topic).isPersistent();
         TopicStats topicStats = admin.topics().getStats(topic);
         SubscriptionStats stats = topicStats.getSubscriptions().get(subName);
         Assert.assertNotNull(stats);
 
         if (setFilter) {
-            Assert.assertEquals(stats.getFilterAcceptedMsgCount(), 100);
+            Assert.assertEquals(stats.getFilterAcceptedMsgCount(), acceptCount);
             if (persistent) {
-                Assert.assertEquals(stats.getFilterRejectedMsgCount(), 100);
+                Assert.assertEquals(stats.getFilterRejectedMsgCount(), rejectedCount);
                 // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount
                 Assert.assertEquals(stats.getFilterProcessedMsgCount(),
                         stats.getFilterAcceptedMsgCount() + stats.getFilterRejectedMsgCount()

From 99a68e40c9dcc0771ddfd73321cc967be6433801 Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Thu, 20 Apr 2023 17:49:27 +0800
Subject: [PATCH 215/404] [fix][client] Move MessageIdAdv to the pulsar-common
 module (#20139)

---
 .../pulsar/client/api/TopicMessageId.java     | 82 +------------------
 .../PulsarClientImplementationBinding.java    |  3 +
 .../pulsar/client/impl/ConsumerImpl.java      |  2 +-
 .../pulsar/client/impl/MessageIdImpl.java     |  4 +-
 ...PulsarClientImplementationBindingImpl.java | 18 +++-
 .../client/impl/TopicMessageIdImpl.java       | 77 +++++++++++++++--
 .../pulsar/client/impl/TopicMessageImpl.java  |  3 +-
 .../client/impl/MessageIdCompareToTest.java   |  5 --
 .../client/impl/TopicMessageIdImplTest.java   | 12 +--
 .../pulsar/client/api/MessageIdAdv.java       |  0
 .../pulsar/client/api/package-info.java       | 22 +++++
 .../kafka/connect/KafkaConnectSinkTest.java   |  4 +-
 .../pulsar/websocket/ConsumerHandler.java     |  6 +-
 13 files changed, 133 insertions(+), 105 deletions(-)
 rename {pulsar-client-api => pulsar-common}/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java (100%)
 create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
index b70267bb0fb8b..4d02a7f4096d6 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
@@ -18,7 +18,7 @@
  */
 package org.apache.pulsar.client.api;
 
-import java.util.BitSet;
+import org.apache.pulsar.client.internal.DefaultImplementation;
 
 /**
  * The MessageId used for a consumer that subscribes multiple topics or partitioned topics.
@@ -45,84 +45,6 @@ static TopicMessageId create(String topic, MessageId messageId) {
         if (messageId instanceof TopicMessageId) {
             return (TopicMessageId) messageId;
         }
-        return new Impl(topic, messageId);
-    }
-
-    /**
-     * The simplest implementation of a TopicMessageId interface.
-     */
-    class Impl implements MessageIdAdv, TopicMessageId {
-        private final String topic;
-        private final MessageIdAdv messageId;
-
-        public Impl(String topic, MessageId messageId) {
-            this.topic = topic;
-            this.messageId = (MessageIdAdv) messageId;
-        }
-
-        @Override
-        public byte[] toByteArray() {
-            return messageId.toByteArray();
-        }
-
-        @Override
-        public String getOwnerTopic() {
-            return topic;
-        }
-
-        @Override
-        public long getLedgerId() {
-            return messageId.getLedgerId();
-        }
-
-        @Override
-        public long getEntryId() {
-            return messageId.getEntryId();
-        }
-
-        @Override
-        public int getPartitionIndex() {
-            return messageId.getPartitionIndex();
-        }
-
-        @Override
-        public int getBatchIndex() {
-            return messageId.getBatchIndex();
-        }
-
-        @Override
-        public int getBatchSize() {
-            return messageId.getBatchSize();
-        }
-
-        @Override
-        public BitSet getAckSet() {
-            return messageId.getAckSet();
-        }
-
-        @Override
-        public MessageIdAdv getFirstChunkMessageId() {
-            return messageId.getFirstChunkMessageId();
-        }
-
-        @Override
-        public int compareTo(MessageId o) {
-            return messageId.compareTo(o);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return messageId.equals(obj);
-        }
-
-        @Override
-        public int hashCode() {
-            return messageId.hashCode();
-        }
-
-        @Override
-        public String toString() {
-            return messageId.toString();
-        }
+        return DefaultImplementation.getDefaultImplementation().newTopicMessageId(topic, messageId);
     }
 }
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java
index 875a793023523..8fd05bff265f1 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java
@@ -37,6 +37,7 @@
 import org.apache.pulsar.client.api.MessagePayloadFactory;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.client.api.schema.GenericRecord;
 import org.apache.pulsar.client.api.schema.GenericSchema;
 import org.apache.pulsar.client.api.schema.RecordSchemaBuilder;
@@ -252,4 +253,6 @@ static byte[] getBytes(ByteBuffer byteBuffer) {
 
     SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, long timestamp,
                                  Map propertiesValue);
+
+    TopicMessageId newTopicMessageId(String topic, MessageId messageId);
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index cc01609319698..199e8a9ae71b4 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -2345,7 +2345,7 @@ public CompletableFuture getLastMessageIdAsync() {
     @Override
     public CompletableFuture> getLastMessageIdsAsync() {
         return getLastMessageIdAsync()
-                .thenApply(msgId -> Collections.singletonList(TopicMessageId.create(topic, msgId)));
+                .thenApply(msgId -> Collections.singletonList(new TopicMessageIdImpl(topic, (MessageIdAdv) msgId)));
     }
 
     public CompletableFuture internalGetLastMessageIdAsync() {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
index 83ee762578390..8cffba44dc5ca 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
@@ -128,7 +128,7 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName)
             throw new IOException(e);
         }
 
-        MessageId messageId;
+        MessageIdAdv messageId;
         if (idData.hasBatchIndex()) {
             if (idData.hasBatchSize()) {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
@@ -143,7 +143,7 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName)
         }
         if (idData.getPartition() > -1 && topicName != null) {
             messageId = new TopicMessageIdImpl(
-                    topicName.getPartition(idData.getPartition()).toString(), topicName.toString(), messageId);
+                    topicName.getPartition(idData.getPartition()).toString(), messageId);
         }
 
         return messageId;
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java
index 1b069c5172dd7..346eb20ef4cc5 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.client.impl;
 
-
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
@@ -35,9 +34,11 @@
 import org.apache.pulsar.client.api.BatcherBuilder;
 import org.apache.pulsar.client.api.ClientBuilder;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.MessagePayloadFactory;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.client.api.schema.GenericRecord;
 import org.apache.pulsar.client.api.schema.GenericSchema;
 import org.apache.pulsar.client.api.schema.RecordSchemaBuilder;
@@ -387,4 +388,19 @@ public SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type,
                                         Map propertiesValue) {
         return new SchemaInfoImpl(name, schema, type, timestamp, propertiesValue);
     }
+
+    @Override
+    public TopicMessageId newTopicMessageId(String topic, MessageId messageId) {
+        final MessageIdAdv messageIdAdv;
+        if (messageId instanceof MessageIdAdv) {
+            messageIdAdv = (MessageIdAdv) messageId;
+        } else {
+            try {
+                messageIdAdv = (MessageIdAdv) MessageId.fromByteArray(messageId.toByteArray());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return new TopicMessageIdImpl(topic, messageIdAdv);
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
index 189dc1c608379..00fe12b62b194 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
@@ -18,15 +18,27 @@
  */
 package org.apache.pulsar.client.impl;
 
+import java.util.BitSet;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.TopicMessageId;
 
-public class TopicMessageIdImpl extends TopicMessageId.Impl {
+public class TopicMessageIdImpl implements MessageIdAdv, TopicMessageId {
 
-    private final String topicName;
+    private final String ownerTopic;
+    private final MessageIdAdv msgId;
+    private final String topicName; // it's never used
 
+    public TopicMessageIdImpl(String topic, MessageIdAdv msgId) {
+        this.ownerTopic = topic;
+        this.msgId = msgId;
+        this.topicName = "";
+    }
+
+    @Deprecated
     public TopicMessageIdImpl(String topicPartitionName, String topicName, MessageId messageId) {
-        super(topicPartitionName, messageId);
+        this.msgId = (MessageIdAdv) messageId;
+        this.ownerTopic = topicPartitionName;
         this.topicName = topicName;
     }
 
@@ -55,11 +67,66 @@ public MessageId getInnerMessageId() {
 
     @Override
     public boolean equals(Object obj) {
-        return super.equals(obj);
+        return msgId.equals(obj);
     }
 
     @Override
     public int hashCode() {
-        return super.hashCode();
+        return msgId.hashCode();
+    }
+
+    @Override
+    public int compareTo(MessageId o) {
+        return msgId.compareTo(o);
+    }
+
+    @Override
+    public byte[] toByteArray() {
+        return msgId.toByteArray();
+    }
+
+    @Override
+    public String getOwnerTopic() {
+        return ownerTopic;
+    }
+
+    @Override
+    public long getLedgerId() {
+        return msgId.getLedgerId();
+    }
+
+    @Override
+    public long getEntryId() {
+        return msgId.getEntryId();
+    }
+
+    @Override
+    public int getPartitionIndex() {
+        return msgId.getPartitionIndex();
+    }
+
+    @Override
+    public int getBatchIndex() {
+        return msgId.getBatchIndex();
+    }
+
+    @Override
+    public int getBatchSize() {
+        return msgId.getBatchSize();
+    }
+
+    @Override
+    public BitSet getAckSet() {
+        return msgId.getAckSet();
+    }
+
+    @Override
+    public MessageIdAdv getFirstChunkMessageId() {
+        return msgId.getFirstChunkMessageId();
+    }
+
+    @Override
+    public String toString() {
+        return msgId.toString();
     }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
index d24ecbd6aa917..1b6cba2f7234d 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
@@ -22,6 +22,7 @@
 import java.util.Optional;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.common.api.EncryptionContext;
 
@@ -42,7 +43,7 @@ public class TopicMessageImpl implements Message {
         this.receivedByconsumer = receivedByConsumer;
 
         this.msg = msg;
-        this.messageId = new TopicMessageIdImpl(topicPartitionName, topicPartitionName, msg.getMessageId());
+        this.messageId = new TopicMessageIdImpl(topicPartitionName, (MessageIdAdv) msg.getMessageId());
     }
 
     /**
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java
index 4f0eca6ea4af8..fd81e9d5790ad 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java
@@ -148,15 +148,12 @@ public void testMessageIdImplCompareToTopicMessageId() {
         MessageIdImpl messageIdImpl = new MessageIdImpl(123L, 345L, 567);
         TopicMessageIdImpl topicMessageId1 = new TopicMessageIdImpl(
             "test-topic-partition-0",
-            "test-topic",
             new BatchMessageIdImpl(123L, 345L, 566, 789));
         TopicMessageIdImpl topicMessageId2 = new TopicMessageIdImpl(
             "test-topic-partition-0",
-            "test-topic",
             new BatchMessageIdImpl(123L, 345L, 567, 789));
         TopicMessageIdImpl topicMessageId3 = new TopicMessageIdImpl(
             "test-topic-partition-0",
-            "test-topic",
             new BatchMessageIdImpl(messageIdImpl));
         assertTrue(messageIdImpl.compareTo(topicMessageId1) > 0, "Expected to be greater than");
         assertTrue(messageIdImpl.compareTo(topicMessageId2) < 0, "Expected to be less than");
@@ -173,11 +170,9 @@ public void testBatchMessageIdImplCompareToTopicMessageId() {
         BatchMessageIdImpl messageIdImpl3 = new BatchMessageIdImpl(123L, 345L, 567, -1);
         TopicMessageIdImpl topicMessageId1 = new TopicMessageIdImpl(
             "test-topic-partition-0",
-            "test-topic",
             new MessageIdImpl(123L, 345L, 566));
         TopicMessageIdImpl topicMessageId2 = new TopicMessageIdImpl(
             "test-topic-partition-0",
-            "test-topic",
             new MessageIdImpl(123L, 345L, 567));
         assertTrue(messageIdImpl1.compareTo(topicMessageId1) > 0, "Expected to be greater than");
         assertTrue(messageIdImpl1.compareTo(topicMessageId2) > 0, "Expected to be greater than");
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java
index d2e2ce9c15c54..daf49f0e7752e 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java
@@ -28,9 +28,9 @@ public class TopicMessageIdImplTest {
     public void hashCodeTest() {
         MessageIdImpl msgId1 = new MessageIdImpl(0, 0, 0);
         MessageIdImpl msgId2 = new BatchMessageIdImpl(1, 1, 1, 1);
-        TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", "topic", msgId1);
-        TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", "topic2", msgId1);
-        TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", "topic", msgId2);
+        TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", msgId1);
+        TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", msgId1);
+        TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", msgId2);
 
         assertEquals(topicMsgId1.hashCode(), topicMsgId1.hashCode());
         assertEquals(topic2MsgId1.hashCode(), topic2MsgId1.hashCode());
@@ -43,9 +43,9 @@ public void hashCodeTest() {
     public void equalsTest() {
         MessageIdImpl msgId1 = new MessageIdImpl(0, 0, 0);
         MessageIdImpl msgId2 = new BatchMessageIdImpl(1, 1, 1, 1);
-        TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", "topic", msgId1);
-        TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", "topic2", msgId1);
-        TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", "topic", msgId2);
+        TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", msgId1);
+        TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", msgId1);
+        TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", msgId2);
 
         assertEquals(topicMsgId1, topicMsgId1);
         assertEquals(topicMsgId1, topic2MsgId1);
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
similarity index 100%
rename from pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
rename to pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java
new file mode 100644
index 0000000000000..3f6d1d56e1032
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Additional helper classes to the pulsar-client-api module.
+ */
+package org.apache.pulsar.client.api;
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index 5410e0bb8d664..1100b13b425b4 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -1572,7 +1572,7 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception {
         assertNull(ref);
 
         ref = KafkaConnectSink.getMessageSequenceRefForBatchMessage(
-                        new TopicMessageIdImpl("topic-0", "topic", new MessageIdImpl(ledgerId, entryId, 0))
+                        new TopicMessageIdImpl("topic-0", new MessageIdImpl(ledgerId, entryId, 0))
         );
         assertNull(ref);
 
@@ -1584,7 +1584,7 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception {
         assertEquals(ref.getBatchIdx(), batchIdx);
 
         ref = KafkaConnectSink.getMessageSequenceRefForBatchMessage(
-                new TopicMessageIdImpl("topic-0", "topic", new BatchMessageIdImpl(ledgerId, entryId, 0, batchIdx))
+                new TopicMessageIdImpl("topic-0", new BatchMessageIdImpl(ledgerId, entryId, 0, batchIdx))
         );
 
         assertEquals(ref.getLedgerId(), ledgerId);
diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
index 579b423339911..c988fd1e70ce3 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
@@ -38,6 +38,7 @@
 import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
 import org.apache.pulsar.client.api.DeadLetterPolicy;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException;
@@ -45,6 +46,7 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.client.impl.ConsumerBuilderImpl;
+import org.apache.pulsar.client.impl.TopicMessageIdImpl;
 import org.apache.pulsar.common.util.Codec;
 import org.apache.pulsar.common.util.DateFormatter;
 import org.apache.pulsar.websocket.data.ConsumerCommand;
@@ -293,8 +295,8 @@ private void checkResumeReceive() {
 
     private void handleAck(ConsumerCommand command) throws IOException {
         // We should have received an ack
-        TopicMessageId msgId = TopicMessageId.create(topic.toString(),
-                MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId)));
+        TopicMessageId msgId = new TopicMessageIdImpl(topic.toString(),
+                (MessageIdAdv) MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId)));
         if (log.isDebugEnabled()) {
             log.debug("[{}/{}] Received ack request of message {} from {} ", consumer.getTopic(),
                     subscription, msgId, getRemote().getInetSocketAddress().toString());

From 00d09cbbd2b3063fecbdc3988b5eef7824f40ce0 Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Thu, 20 Apr 2023 18:11:52 +0800
Subject: [PATCH 216/404] [fix][txn] Fix transaction is not aborted when send
 or ACK failed (#20055)

---
 .../transaction/TransactionProduceTest.java   | 35 ++++++++--
 .../impl/transaction/TransactionImpl.java     | 70 +++++++++----------
 2 files changed, 62 insertions(+), 43 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java
index cdbb1563280b4..ddd8cf0790321 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java
@@ -26,6 +26,8 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import lombok.Cleanup;
@@ -43,6 +45,7 @@
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.transaction.Transaction;
@@ -51,10 +54,11 @@
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.protocol.Commands;
+import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException;
 import org.awaitility.Awaitility;
 import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 /**
@@ -70,7 +74,7 @@ public class TransactionProduceTest extends TransactionTestBase {
     private static final String ACK_COMMIT_TOPIC = NAMESPACE1 + "/ack-commit";
     private static final String ACK_ABORT_TOPIC = NAMESPACE1 + "/ack-abort";
     private static final int NUM_PARTITIONS = 16;
-    @BeforeMethod
+    @BeforeClass
     protected void setup() throws Exception {
         setUpBase(1, NUM_PARTITIONS, PRODUCE_COMMIT_TOPIC, TOPIC_PARTITION);
         admin.topics().createPartitionedTopic(PRODUCE_ABORT_TOPIC, TOPIC_PARTITION);
@@ -78,7 +82,7 @@ protected void setup() throws Exception {
         admin.topics().createPartitionedTopic(ACK_ABORT_TOPIC, TOPIC_PARTITION);
     }
 
-    @AfterMethod(alwaysRun = true)
+    @AfterClass(alwaysRun = true)
     protected void cleanup() throws Exception {
         super.internalCleanup();
     }
@@ -369,5 +373,26 @@ private int getPendingAckCount(String topic, String subscriptionName) throws Exc
         return pendingAckCount;
     }
 
-
+    @Test
+    public void testCommitFailure() throws Exception {
+        Transaction txn = pulsarClient.newTransaction().build().get();
+        final String topic = NAMESPACE1 + "/test-commit-failure";
+        @Cleanup
+        final Producer producer = pulsarClient.newProducer().topic(topic).create();
+        producer.newMessage(txn).value(new byte[1024 * 1024 * 10]).sendAsync();
+        try {
+            txn.commit().get();
+            Assert.fail();
+        } catch (ExecutionException e) {
+            Assert.assertTrue(e.getCause() instanceof PulsarClientException.TransactionHasOperationFailedException);
+            Assert.assertEquals(txn.getState(), Transaction.State.ABORTED);
+        }
+        try {
+            getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txn.getTxnID())
+                    .getNow(null);
+            Assert.fail();
+        } catch (CompletionException e) {
+            Assert.assertTrue(e.getCause() instanceof CoordinatorException.TransactionNotFoundException);
+        }
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java
index b7e085ed82a85..d1260ba045e6d 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java
@@ -21,6 +21,7 @@
 import com.google.common.collect.Lists;
 import io.netty.util.Timeout;
 import io.netty.util.TimerTask;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -186,13 +187,14 @@ public void registerAckOp(CompletableFuture newAckFuture) {
     @Override
     public CompletableFuture commit() {
         timeout.cancel();
-        return checkIfOpenOrCommitting().thenCompose((value) -> {
+        return checkState(State.OPEN, State.COMMITTING).thenCompose((value) -> {
             CompletableFuture commitFuture = new CompletableFuture<>();
             this.state = State.COMMITTING;
             opFuture.whenComplete((v, e) -> {
                 if (hasOpsFailed) {
-                    abort().whenComplete((vx, ex) -> commitFuture.completeExceptionally(new PulsarClientException
-                            .TransactionHasOperationFailedException()));
+                    checkState(State.COMMITTING).thenCompose(__ -> internalAbort()).whenComplete((vx, ex) ->
+                            commitFuture.completeExceptionally(
+                                    new PulsarClientException.TransactionHasOperationFailedException()));
                 } else {
                     tcClient.commitAsync(txnId)
                             .whenComplete((vx, ex) -> {
@@ -216,28 +218,30 @@ public CompletableFuture commit() {
     @Override
     public CompletableFuture abort() {
         timeout.cancel();
-        return checkIfOpenOrAborting().thenCompose(value -> {
-            CompletableFuture abortFuture = new CompletableFuture<>();
-            this.state = State.ABORTING;
-            opFuture.whenComplete((v, e) -> {
-                tcClient.abortAsync(txnId).whenComplete((vx, ex) -> {
+        return checkState(State.OPEN, State.ABORTING).thenCompose(__ -> internalAbort());
+    }
 
-                    if (ex != null) {
-                        if (ex instanceof TransactionNotFoundException
-                                || ex instanceof InvalidTxnStatusException) {
-                            this.state = State.ERROR;
-                        }
-                        abortFuture.completeExceptionally(ex);
-                    } else {
-                        this.state = State.ABORTED;
-                        abortFuture.complete(null);
+    private CompletableFuture internalAbort() {
+        CompletableFuture abortFuture = new CompletableFuture<>();
+        this.state = State.ABORTING;
+        opFuture.whenComplete((v, e) -> {
+            tcClient.abortAsync(txnId).whenComplete((vx, ex) -> {
+
+                if (ex != null) {
+                    if (ex instanceof TransactionNotFoundException
+                            || ex instanceof InvalidTxnStatusException) {
+                        this.state = State.ERROR;
                     }
+                    abortFuture.completeExceptionally(ex);
+                } else {
+                    this.state = State.ABORTED;
+                    abortFuture.complete(null);
+                }
 
-                });
             });
-
-            return abortFuture;
         });
+
+        return abortFuture;
     }
 
     @Override
@@ -261,25 +265,15 @@ public  boolean checkIfOpen(CompletableFuture completableFuture) {
         }
     }
 
-    private CompletableFuture checkIfOpenOrCommitting() {
-        if (state == State.OPEN || state == State.COMMITTING) {
-            return CompletableFuture.completedFuture(null);
-        } else {
-            return invalidTxnStatusFuture();
-        }
-    }
-
-    private CompletableFuture checkIfOpenOrAborting() {
-        if (state == State.OPEN || state == State.ABORTING) {
-            return CompletableFuture.completedFuture(null);
-        } else {
-            return invalidTxnStatusFuture();
+    private CompletableFuture checkState(State... expectedStates) {
+        final State actualState = STATE_UPDATE.get(this);
+        for (State expectedState : expectedStates) {
+            if (actualState == expectedState) {
+                return CompletableFuture.completedFuture(null);
+            }
         }
-    }
-
-    private CompletableFuture invalidTxnStatusFuture() {
         return FutureUtil.failedFuture(new InvalidTxnStatusException("[" + txnIdMostBits + ":"
-                + txnIdLeastBits + "] with unexpected state : "
-                + state.name() + ", expect " + State.OPEN + " state!"));
+                + txnIdLeastBits + "] with unexpected state: " + actualState.name() + ", expect: "
+                + Arrays.toString(expectedStates)));
     }
 }

From 2b41e4eafb1cba0e548dd90df60e8cdbb24cd490 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Thu, 20 Apr 2023 20:13:50 +0800
Subject: [PATCH 217/404] [improve] [broker] Pin AppendIndexMetadataInterceptor
 to field in `ManagedLedgerInterceptorImpl` (#20112)

---
 .../ManagedLedgerInterceptorImpl.java         | 60 +++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java
index 30713c91907ef..02c6c575fd919 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java
@@ -40,12 +40,27 @@ public class ManagedLedgerInterceptorImpl implements ManagedLedgerInterceptor {
     private static final Logger log = LoggerFactory.getLogger(ManagedLedgerInterceptorImpl.class);
     private static final String INDEX = "index";
     private final Set brokerEntryMetadataInterceptors;
+
+    private final AppendIndexMetadataInterceptor appendIndexMetadataInterceptor;
     private final Set inputProcessors;
     private final Set outputProcessors;
 
     public ManagedLedgerInterceptorImpl(Set brokerEntryMetadataInterceptors,
                                         Set brokerEntryPayloadProcessors) {
         this.brokerEntryMetadataInterceptors = brokerEntryMetadataInterceptors;
+
+        // save appendIndexMetadataInterceptor to field
+        AppendIndexMetadataInterceptor appendIndexMetadataInterceptor = null;
+
+        for (BrokerEntryMetadataInterceptor interceptor : this.brokerEntryMetadataInterceptors) {
+            if (interceptor instanceof AppendIndexMetadataInterceptor) {
+                appendIndexMetadataInterceptor = (AppendIndexMetadataInterceptor) interceptor;
+                break;
+            }
+        }
+
+        this.appendIndexMetadataInterceptor = appendIndexMetadataInterceptor;
+
         if (brokerEntryPayloadProcessors != null) {
             this.inputProcessors = new LinkedHashSet<>();
             this.outputProcessors = new LinkedHashSet<>();
@@ -61,12 +76,11 @@ public ManagedLedgerInterceptorImpl(Set brokerEn
 
     public long getIndex() {
         long index = -1;
-        for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
-            if (interceptor instanceof AppendIndexMetadataInterceptor) {
-                index = ((AppendIndexMetadataInterceptor) interceptor).getIndex();
-                break;
-            }
+
+        if (appendIndexMetadataInterceptor != null) {
+            return appendIndexMetadataInterceptor.getIndex();
         }
+
         return index;
     }
 
@@ -81,10 +95,8 @@ public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) {
 
     @Override
     public void afterFailedAddEntry(int numberOfMessages) {
-        for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
-            if (interceptor instanceof AppendIndexMetadataInterceptor) {
-                ((AppendIndexMetadataInterceptor) interceptor).decreaseWithNumberOfMessages(numberOfMessages);
-            }
+        if (appendIndexMetadataInterceptor != null) {
+            appendIndexMetadataInterceptor.decreaseWithNumberOfMessages(numberOfMessages);
         }
     }
 
@@ -95,12 +107,9 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa
         }
 
         if (propertiesMap.containsKey(INDEX)) {
-            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
-                if (interceptor instanceof AppendIndexMetadataInterceptor) {
-                  ((AppendIndexMetadataInterceptor) interceptor)
-                          .recoveryIndexGenerator(Long.parseLong(propertiesMap.get(INDEX)));
-                  break;
-                }
+            if (appendIndexMetadataInterceptor != null) {
+                appendIndexMetadataInterceptor.recoveryIndexGenerator(
+                        Long.parseLong(propertiesMap.get(INDEX)));
             }
         }
     }
@@ -108,8 +117,7 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa
     @Override
     public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) {
         CompletableFuture promise = new CompletableFuture<>();
-        boolean hasAppendIndexMetadataInterceptor = brokerEntryMetadataInterceptors.stream()
-                .anyMatch(interceptor -> interceptor instanceof AppendIndexMetadataInterceptor);
+        boolean hasAppendIndexMetadataInterceptor = appendIndexMetadataInterceptor != null;
         if (hasAppendIndexMetadataInterceptor && lh.getLastAddConfirmed() >= 0) {
             lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> {
                 if (ex != null) {
@@ -122,14 +130,9 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name,
                             if (ledgerEntry != null) {
                                 BrokerEntryMetadata brokerEntryMetadata =
                                         Commands.parseBrokerEntryMetadataIfExist(ledgerEntry.getEntryBuffer());
-                                for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
-                                    if (interceptor instanceof AppendIndexMetadataInterceptor) {
-                                        if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) {
-                                            ((AppendIndexMetadataInterceptor) interceptor)
-                                                    .recoveryIndexGenerator(brokerEntryMetadata.getIndex());
-                                        }
-                                        break;
-                                    }
+                                if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) {
+                                    appendIndexMetadataInterceptor.recoveryIndexGenerator(
+                                            brokerEntryMetadata.getIndex());
                                 }
                             }
                             entries.close();
@@ -153,11 +156,8 @@ public CompletableFuture onManagedLedgerLastLedgerInitialize(String name,
 
     @Override
     public void onUpdateManagedLedgerInfo(Map propertiesMap) {
-        for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
-            if (interceptor instanceof AppendIndexMetadataInterceptor) {
-                propertiesMap.put(INDEX, String.valueOf(((AppendIndexMetadataInterceptor) interceptor).getIndex()));
-                break;
-            }
+        if (appendIndexMetadataInterceptor != null) {
+            propertiesMap.put(INDEX, String.valueOf(appendIndexMetadataInterceptor.getIndex()));
         }
     }
 

From e1a3fa8f6d9e3cb4e4357ee40263f55e1c1491b8 Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Fri, 21 Apr 2023 09:39:57 +0800
Subject: [PATCH 218/404] [improve][broker] Support disabling delayed bucket
 merging. (#20155)

---
 conf/broker.conf                                             | 3 ++-
 conf/standalone.conf                                         | 3 ++-
 .../java/org/apache/pulsar/broker/ServiceConfiguration.java  | 5 +++--
 .../broker/delayed/bucket/BucketDelayedDeliveryTracker.java  | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 292b1ef1683b7..89d2d85200448 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -577,7 +577,8 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
 
 # The max number of delayed message index bucket,
 # after reaching the max buckets limitation, the adjacent buckets will be merged.
-delayedDeliveryMaxNumBuckets=50
+# (disable with value -1)
+delayedDeliveryMaxNumBuckets=-1
 
 # Size of the lookahead window to use when detecting if all the messages in the topic
 # have a fixed delay.
diff --git a/conf/standalone.conf b/conf/standalone.conf
index f141946c29f4e..63bc7a29ae6da 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -1264,4 +1264,5 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
 
 # The max number of delayed message index bucket,
 # after reaching the max buckets limitation, the adjacent buckets will be merged.
-delayedDeliveryMaxNumBuckets=50
+# (disable with value -1)
+delayedDeliveryMaxNumBuckets=-1
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index b31c86560f848..2c48310f96482 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -372,8 +372,9 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
             The max number of delayed message index bucket, \
-            after reaching the max buckets limitation, the adjacent buckets will be merged.""")
-    private int delayedDeliveryMaxNumBuckets = 50;
+            after reaching the max buckets limitation, the adjacent buckets will be merged.\
+            (disable with value -1)""")
+    private int delayedDeliveryMaxNumBuckets = -1;
 
     @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use "
             + "when detecting if all the messages in the topic have a fixed delay. "
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 5d1dc2e27033a..b17387e276e2b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -335,7 +335,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
             afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime);
             lastMutableBucket.resetLastMutableBucketRange();
 
-            if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) {
+            if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets) {
                 asyncMergeBucketSnapshot();
             }
         }

From a20d5e96acc91f8099845e8924e84cb0b5632f3f Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Fri, 21 Apr 2023 09:51:55 +0800
Subject: [PATCH 219/404] [improve][broker] Optimization protobuf code in the
 bucket delayed tracker (#20158)

---
 pulsar-broker/pom.xml                         |  2 +
 .../BookkeeperBucketSnapshotStorage.java      | 24 ++---
 .../pulsar/broker/delayed/bucket/Bucket.java  |  7 +-
 .../bucket/BucketDelayedDeliveryTracker.java  | 13 ++-
 .../delayed/bucket/BucketSnapshotStorage.java |  4 +-
 .../CombinedSegmentDelayedIndexQueue.java     | 22 ++++-
 .../delayed/bucket/DelayedIndexQueue.java     | 12 ++-
 .../delayed/bucket/ImmutableBucket.java       | 19 ++--
 .../broker/delayed/bucket/MutableBucket.java  | 29 +++---
 .../TripleLongPriorityDelayedIndexQueue.java  | 25 +++--
 ...> DelayedMessageIndexBucketMetadata.proto} | 11 +--
 .../DelayedMessageIndexBucketSegment.proto    | 35 +++++++
 .../BookkeeperBucketSnapshotStorageTest.java  | 95 +++++++++----------
 .../delayed/MockBucketSnapshotStorage.java    | 14 +--
 .../delayed/bucket/DelayedIndexQueueTest.java | 70 ++++++--------
 15 files changed, 210 insertions(+), 172 deletions(-)
 rename pulsar-broker/src/main/proto/{DelayedMessageIndexBucketSnapshotFormat.proto => DelayedMessageIndexBucketMetadata.proto} (85%)
 create mode 100644 pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto

diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 7442d95e467e6..6e98386e9e931 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -566,6 +566,7 @@
           
             **/ResourceUsage.proto
             **/TransactionPendingAck.proto
+            **/DelayedMessageIndexBucketSegment.proto
           
         
         
@@ -610,6 +611,7 @@
           
             ${project.basedir}/src/main/proto/TransactionPendingAck.proto
             ${project.basedir}/src/main/proto/ResourceUsage.proto
+            ${project.basedir}/src/main/proto/DelayedMessageIndexBucketSegment.proto
           
           generated-sources/lightproto/java
           generated-sources/lightproto/java
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 18a4c322f7b27..040bbbc586f49 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -19,7 +19,7 @@
 package org.apache.pulsar.broker.delayed.bucket;
 
 import com.google.protobuf.InvalidProtocolBufferException;
-import java.io.IOException;
+import io.netty.buffer.ByteBuf;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
@@ -36,8 +36,8 @@
 import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 import org.apache.pulsar.common.util.FutureUtil;
 
 @Slf4j
@@ -126,7 +126,8 @@ private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle,
 
     private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) {
         try {
-            return SnapshotMetadata.parseFrom(ledgerEntry.getEntry());
+            ByteBuf entryBuffer = ledgerEntry.getEntryBuffer();
+            return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer());
         } catch (InvalidProtocolBufferException e) {
             throw new BucketSnapshotSerializationException(e);
         }
@@ -134,15 +135,14 @@ private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) {
 
     private List parseSnapshotSegmentEntries(Enumeration entryEnumeration) {
         List snapshotMetadataList = new ArrayList<>();
-        try {
-            while (entryEnumeration.hasMoreElements()) {
-                LedgerEntry ledgerEntry = entryEnumeration.nextElement();
-                snapshotMetadataList.add(SnapshotSegment.parseFrom(ledgerEntry.getEntry()));
-            }
-            return snapshotMetadataList;
-        } catch (IOException e) {
-            throw new BucketSnapshotSerializationException(e);
+        while (entryEnumeration.hasMoreElements()) {
+            LedgerEntry ledgerEntry = entryEnumeration.nextElement();
+            SnapshotSegment snapshotSegment = new SnapshotSegment();
+            ByteBuf entryBuffer = ledgerEntry.getEntryBuffer();
+            snapshotSegment.parseFrom(entryBuffer, entryBuffer.readableBytes());
+            snapshotMetadataList.add(snapshotSegment);
         }
+        return snapshotMetadataList;
     }
 
     @NotNull
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index 4d7d3aa512be6..a1693b1553d97 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -31,7 +31,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 import org.apache.pulsar.common.util.Codec;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.roaringbitmap.RoaringBitmap;
@@ -132,8 +133,8 @@ long getAndUpdateBucketId() {
     }
 
     CompletableFuture asyncSaveBucketSnapshot(
-            ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata,
-            List bucketSnapshotSegments) {
+            ImmutableBucket bucket, SnapshotMetadata snapshotMetadata,
+            List bucketSnapshotSegments) {
         final String bucketKey = bucket.bucketKey();
         final String cursorName = Codec.decode(cursor.getName());
         final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index b17387e276e2b..c90064c9137bb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -53,8 +53,8 @@
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
 import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -286,8 +286,7 @@ private void afterCreateImmutableBucket(Pair immu
                     // Put indexes back into the shared queue and downgrade to memory mode
                     synchronized (BucketDelayedDeliveryTracker.this) {
                         immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> {
-                            for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment :
-                                    snapshotSegments) {
+                            for (SnapshotSegment snapshotSegment : snapshotSegments) {
                                 for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) {
                                     sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(),
                                             delayedIndex.getLedgerId(), delayedIndex.getEntryId());
@@ -450,7 +449,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List>> getRemainFutures =
+            List>> getRemainFutures =
                     buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList();
 
             return FutureUtil.waitForAll(getRemainFutures)
@@ -601,11 +600,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                             bucket.asyncDeleteBucketSnapshot(stats);
                             return;
                         }
-                        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex
+                        DelayedIndex
                                 lastDelayedIndex = indexList.get(indexList.size() - 1);
                         this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(),
                                 lastDelayedIndex.getEntryId(), bucket);
-                        for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : indexList) {
+                        for (DelayedIndex index : indexList) {
                             sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(),
                                     index.getEntryId());
                         }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
index 51c89bed47af2..7464ef9cd3f63 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
@@ -20,8 +20,8 @@
 
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 
 public interface BucketSnapshotStorage {
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
index 5655a26878296..006938e9ed271 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
@@ -24,8 +24,8 @@
 import java.util.PriorityQueue;
 import javax.annotation.concurrent.NotThreadSafe;
 import lombok.AllArgsConstructor;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 
 @NotThreadSafe
 class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue {
@@ -40,8 +40,8 @@ static class Node {
     }
 
     private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare(
-            node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor),
-            node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor));
+            node1.segmentList.get(node1.segmentListCursor).getIndexeAt(node1.segmentCursor),
+            node2.segmentList.get(node2.segmentListCursor).getIndexeAt(node2.segmentCursor));
 
     private final PriorityQueue kpq;
 
@@ -77,7 +77,7 @@ private DelayedIndex getValue(boolean needAdvanceCursor) {
         Objects.requireNonNull(node);
 
         SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor);
-        DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor);
+        DelayedIndex delayedIndex = snapshotSegment.getIndexeAt(node.segmentCursor);
         if (!needAdvanceCursor) {
             return delayedIndex;
         }
@@ -104,4 +104,16 @@ private DelayedIndex getValue(boolean needAdvanceCursor) {
 
         return delayedIndex;
     }
+
+    @Override
+    public void popToObject(DelayedIndex delayedIndex) {
+        DelayedIndex value = getValue(true);
+        delayedIndex.copyFrom(value);
+    }
+
+    @Override
+    public long peekTimestamp() {
+        DelayedIndex value = getValue(false);
+        return value.getTimestamp();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java
index dee476c376ec9..f1209a3137a45 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java
@@ -20,10 +20,10 @@
 
 import java.util.Comparator;
 import java.util.Objects;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
 
 interface DelayedIndexQueue {
-    Comparator COMPARATOR = (o1, o2) ->  {
+    Comparator COMPARATOR = (o1, o2) ->  {
         if (!Objects.equals(o1.getTimestamp(), o2.getTimestamp())) {
             return Long.compare(o1.getTimestamp(), o2.getTimestamp());
         } else if (!Objects.equals(o1.getLedgerId(), o2.getLedgerId())) {
@@ -35,7 +35,11 @@ interface DelayedIndexQueue {
 
     boolean isEmpty();
 
-    DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek();
+    DelayedIndex peek();
 
-    DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop();
+    DelayedIndex pop();
+
+    void popToObject(DelayedIndex delayedIndex);
+
+    long peekTimestamp();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index 57de5c84fcd82..0932f51f350ce 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -32,9 +32,9 @@
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.mutable.MutableLong;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.roaringbitmap.InvalidRoaringFormat;
 import org.roaringbitmap.RoaringBitmap;
@@ -43,7 +43,7 @@
 class ImmutableBucket extends Bucket {
 
     @Setter
-    private List snapshotSegments;
+    private List snapshotSegments;
 
     boolean merging = false;
 
@@ -55,7 +55,7 @@ class ImmutableBucket extends Bucket {
         super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId);
     }
 
-    public Optional> getSnapshotSegments() {
+    public Optional> getSnapshotSegments() {
         return Optional.ofNullable(snapshotSegments);
     }
 
@@ -84,7 +84,7 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
                         }
                     }), BucketSnapshotPersistenceException.class, MaxRetryTimes)
                     .thenApply(snapshotMetadata -> {
-                        List metadataList =
+                        List metadataList =
                                 snapshotMetadata.getMetadataListList();
 
                         // Skip all already reach schedule time snapshot segments
@@ -125,10 +125,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
                             return Collections.emptyList();
                         }
 
-                        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment =
+                        SnapshotSegment snapshotSegment =
                                 bucketSnapshotSegments.get(0);
-                        List indexList =
-                                snapshotSegment.getIndexesList();
+                        List indexList = snapshotSegment.getIndexesList();
                         this.setCurrentSegmentEntryId(nextSegmentEntryId);
                         if (isRecover) {
                             this.asyncUpdateSnapshotLength();
@@ -171,7 +170,7 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex,
         setNumberBucketDelayedMessages(numberMessages.getValue());
     }
 
-    CompletableFuture> getRemainSnapshotSegment() {
+    CompletableFuture> getRemainSnapshotSegment() {
         int nextSegmentEntryId = currentSegmentEntryId + 1;
         if (nextSegmentEntryId > lastSegmentEntryId) {
             return CompletableFuture.completedFuture(Collections.emptyList());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index f404d5d02c15a..b7e9e68f1bdc7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -30,10 +30,10 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 import org.roaringbitmap.RoaringBitmap;
@@ -75,14 +75,16 @@ Pair createImmutableBucketAndAsyncPersistent(
         List bucketSnapshotSegments = new ArrayList<>();
         List segmentMetadataList = new ArrayList<>();
         Map bitMap = new HashMap<>();
-        SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder();
+        SnapshotSegment snapshotSegment = new SnapshotSegment();
         SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder();
 
         List firstScheduleTimestamps = new ArrayList<>();
         long currentTimestampUpperLimit = 0;
         long currentFirstTimestamp = 0L;
         while (!delayedIndexQueue.isEmpty()) {
-            DelayedIndex delayedIndex = delayedIndexQueue.peek();
+            DelayedIndex delayedIndex = snapshotSegment.addIndexe();
+            delayedIndexQueue.popToObject(delayedIndex);
+
             long timestamp = delayedIndex.getTimestamp();
             if (currentTimestampUpperLimit == 0) {
                 currentFirstTimestamp = timestamp;
@@ -100,16 +102,13 @@ Pair createImmutableBucketAndAsyncPersistent(
                 sharedQueue.add(timestamp, ledgerId, entryId);
             }
 
-            delayedIndexQueue.pop();
             numMessages++;
 
             bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1);
 
-            snapshotSegmentBuilder.addIndexes(delayedIndex);
-
-            if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit
+            if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit
                     || (maxIndexesPerBucketSnapshotSegment != -1
-                    && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
+                    && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
                 segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp);
                 segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp);
                 currentTimestampUpperLimit = 0;
@@ -129,8 +128,8 @@ Pair createImmutableBucketAndAsyncPersistent(
                 segmentMetadataList.add(segmentMetadataBuilder.build());
                 segmentMetadataBuilder.clear();
 
-                bucketSnapshotSegments.add(snapshotSegmentBuilder.build());
-                snapshotSegmentBuilder.clear();
+                bucketSnapshotSegments.add(snapshotSegment);
+                snapshotSegment = new SnapshotSegment();
             }
         }
 
@@ -153,8 +152,8 @@ Pair createImmutableBucketAndAsyncPersistent(
 
         // Add the first snapshot segment last message to snapshotSegmentLastMessageTable
         checkArgument(!bucketSnapshotSegments.isEmpty());
-        SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0);
-        DelayedIndex lastDelayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1);
+        SnapshotSegment firstSnapshotSegment = bucketSnapshotSegments.get(0);
+        DelayedIndex lastDelayedIndex = firstSnapshotSegment.getIndexeAt(firstSnapshotSegment.getIndexesCount() - 1);
         Pair result = Pair.of(bucket, lastDelayedIndex);
 
         CompletableFuture future = asyncSaveBucketSnapshot(bucket,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java
index b8d54bd78b428..4faee3b17f17f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java
@@ -19,7 +19,7 @@
 package org.apache.pulsar.broker.delayed.bucket;
 
 import javax.annotation.concurrent.NotThreadSafe;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 
 @NotThreadSafe
@@ -41,17 +41,28 @@ public boolean isEmpty() {
     }
 
     @Override
-    public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek() {
-        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex =
-                DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setTimestamp(queue.peekN1())
-                        .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()).build();
+    public DelayedIndex peek() {
+        DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(queue.peekN1())
+                .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3());
         return delayedIndex;
     }
 
     @Override
-    public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop() {
-        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek = peek();
+    public DelayedIndex pop() {
+        DelayedIndex peek = peek();
         queue.pop();
         return peek;
     }
+
+    @Override
+    public void popToObject(DelayedIndex delayedIndex) {
+        delayedIndex.setTimestamp(queue.peekN1())
+                .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3());
+        queue.pop();
+    }
+
+    @Override
+    public long peekTimestamp() {
+        return queue.peekN1();
+    }
 }
diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto
similarity index 85%
rename from pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
rename to pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto
index 6996b860c5249..01b770c567d09 100644
--- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
+++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto
@@ -21,12 +21,7 @@ syntax = "proto2";
 package pulsar.delay;
 option java_package = "org.apache.pulsar.broker.delayed.proto";
 option optimize_for = SPEED;
-
-message DelayedIndex {
-    required uint64 timestamp = 1;
-    required uint64 ledger_id = 2;
-    required uint64 entry_id = 3;
-}
+option java_multiple_files = true;
 
 message SnapshotSegmentMetadata {
     map delayed_index_bit_map = 1;
@@ -34,10 +29,6 @@ message SnapshotSegmentMetadata {
     required uint64 min_schedule_timestamp = 3;
 }
 
-message SnapshotSegment {
-    repeated DelayedIndex indexes = 1;
-}
-
 message SnapshotMetadata {
     repeated SnapshotSegmentMetadata metadata_list = 1;
 }
diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto
new file mode 100644
index 0000000000000..633d6a8f1615c
--- /dev/null
+++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+syntax = "proto2";
+
+package pulsar.delay;
+option java_package = "org.apache.pulsar.broker.delayed.proto";
+option optimize_for = SPEED;
+option java_multiple_files = true;
+
+message DelayedIndex {
+    required uint64 timestamp = 1;
+    required uint64 ledger_id = 2;
+    required uint64 entry_id = 3;
+}
+
+message SnapshotSegment {
+    repeated DelayedIndex indexes = 1;
+    map delayed_index_bit_map = 2;
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
index 7cb6b8d5865bb..d26f38fa2bc2b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
@@ -29,7 +29,10 @@
 import java.util.concurrent.ExecutionException;
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
@@ -60,9 +63,8 @@ protected void cleanup() throws Exception {
 
     @Test
     public void testCreateSnapshot() throws ExecutionException, InterruptedException {
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build();
-        List bucketSnapshotSegments = new ArrayList<>();
+        SnapshotMetadata snapshotMetadata = SnapshotMetadata.newBuilder().build();
+        List bucketSnapshotSegments = new ArrayList<>();
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
                         bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
@@ -72,24 +74,23 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException
 
     @Test
     public void testGetSnapshot() throws ExecutionException, InterruptedException {
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+        SnapshotSegmentMetadata segmentMetadata =
+                SnapshotSegmentMetadata.newBuilder()
                         .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder()
+        SnapshotMetadata snapshotMetadata =
+                SnapshotMetadata.newBuilder()
                         .addMetadataList(segmentMetadata)
                         .build();
-        List bucketSnapshotSegments = new ArrayList<>();
+        List bucketSnapshotSegments = new ArrayList<>();
 
         long timeMillis = System.currentTimeMillis();
-        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex =
-                DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L)
-                        .setTimestamp(timeMillis).build();
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build();
+        DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L)
+                        .setTimestamp(timeMillis);
+        SnapshotSegment snapshotSegment = new SnapshotSegment();
+        snapshotSegment.addIndexe().copyFrom(delayedIndex);
         bucketSnapshotSegments.add(snapshotSegment);
         bucketSnapshotSegments.add(snapshotSegment);
 
@@ -99,13 +100,13 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException {
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
-        CompletableFuture> bucketSnapshotSegment =
+        CompletableFuture> bucketSnapshotSegment =
                 bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3);
 
-        List snapshotSegments = bucketSnapshotSegment.get();
+        List snapshotSegments = bucketSnapshotSegment.get();
         Assert.assertEquals(2, snapshotSegments.size());
-        for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment segment : snapshotSegments) {
-            for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : segment.getIndexesList()) {
+        for (SnapshotSegment segment : snapshotSegments) {
+            for (DelayedIndex index : segment.getIndexesList()) {
                 Assert.assertEquals(100L, index.getLedgerId());
                 Assert.assertEquals(10L, index.getEntryId());
                 Assert.assertEquals(timeMillis, index.getTimestamp());
@@ -121,17 +122,17 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
         map.put(100L, ByteString.copyFrom("test1", StandardCharsets.UTF_8));
         map.put(200L, ByteString.copyFrom("test2", StandardCharsets.UTF_8));
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+        SnapshotSegmentMetadata segmentMetadata =
+                SnapshotSegmentMetadata.newBuilder()
                         .setMaxScheduleTimestamp(timeMillis)
                         .setMinScheduleTimestamp(timeMillis)
                         .putAllDelayedIndexBitMap(map).build();
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder()
+        SnapshotMetadata snapshotMetadata =
+                SnapshotMetadata.newBuilder()
                         .addMetadataList(segmentMetadata)
                         .build();
-        List bucketSnapshotSegments = new ArrayList<>();
+        List bucketSnapshotSegments = new ArrayList<>();
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
@@ -139,10 +140,10 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata bucketSnapshotMetadata =
+        SnapshotMetadata bucketSnapshotMetadata =
                 bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).get();
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata metadata =
+        SnapshotSegmentMetadata metadata =
                 bucketSnapshotMetadata.getMetadataList(0);
 
         Assert.assertEquals(timeMillis, metadata.getMaxScheduleTimestamp());
@@ -152,9 +153,9 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
 
     @Test
     public void testDeleteSnapshot() throws ExecutionException, InterruptedException {
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build();
-        List bucketSnapshotSegments = new ArrayList<>();
+        SnapshotMetadata snapshotMetadata =
+                SnapshotMetadata.newBuilder().build();
+        List bucketSnapshotSegments = new ArrayList<>();
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
                         bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
@@ -173,24 +174,22 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException
 
     @Test
     public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException {
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+        SnapshotSegmentMetadata segmentMetadata =
+                SnapshotSegmentMetadata.newBuilder()
                         .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder()
+        SnapshotMetadata snapshotMetadata =
+                SnapshotMetadata.newBuilder()
                         .addMetadataList(segmentMetadata)
                         .build();
-        List bucketSnapshotSegments = new ArrayList<>();
+        List bucketSnapshotSegments = new ArrayList<>();
 
         long timeMillis = System.currentTimeMillis();
-        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex =
-                DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L)
-                        .setTimestamp(timeMillis).build();
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build();
+        DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis);
+        SnapshotSegment snapshotSegment = new SnapshotSegment();
+        snapshotSegment.addIndexe().copyFrom(delayedIndex);
         bucketSnapshotSegments.add(snapshotSegment);
         bucketSnapshotSegments.add(snapshotSegment);
 
@@ -207,24 +206,22 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted
 
     @Test
     public void testConcurrencyGet() throws ExecutionException, InterruptedException {
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+        SnapshotSegmentMetadata segmentMetadata =
+                SnapshotSegmentMetadata.newBuilder()
                         .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder()
+        SnapshotMetadata snapshotMetadata =
+                SnapshotMetadata.newBuilder()
                         .addMetadataList(segmentMetadata)
                         .build();
-        List bucketSnapshotSegments = new ArrayList<>();
+        List bucketSnapshotSegments = new ArrayList<>();
 
         long timeMillis = System.currentTimeMillis();
-        DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex =
-                DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L)
-                        .setTimestamp(timeMillis).build();
-        DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment =
-                DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build();
+        DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis);
+        SnapshotSegment snapshotSegment = new SnapshotSegment();
+        snapshotSegment.addIndexe().copyFrom(delayedIndex);
         bucketSnapshotSegments.add(snapshotSegment);
         bucketSnapshotSegments.add(snapshotSegment);
 
@@ -237,7 +234,7 @@ public void testConcurrencyGet() throws ExecutionException, InterruptedException
         List> futures = new ArrayList<>();
         for (int i = 0; i < 100; i++) {
             CompletableFuture future0 = CompletableFuture.runAsync(() -> {
-                List list =
+                List list =
                         bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join();
                 Assert.assertTrue(list.size() > 0);
             });
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
index 9e924bdeda341..dc1c0e09ca276 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
@@ -36,8 +36,8 @@
 import java.util.concurrent.atomic.AtomicLong;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 import org.apache.pulsar.common.util.FutureUtil;
 
 @Slf4j
@@ -139,13 +139,9 @@ public CompletableFuture> getBucketSnapshotSegment(long bu
             long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size());
             for (int i = (int) firstSegmentEntryId; i <= lastEntryId ; i++) {
                 ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(i);
-                SnapshotSegment snapshotSegment;
-                try {
-                    snapshotSegment = SnapshotSegment.parseFrom(byteBuf.nioBuffer());
-                    snapshotSegments.add(snapshotSegment);
-                } catch (InvalidProtocolBufferException e) {
-                    throw new RuntimeException(e);
-                }
+                SnapshotSegment snapshotSegment = new SnapshotSegment();
+                snapshotSegment.parseFrom(byteBuf, byteBuf.readableBytes());
+                snapshotSegments.add(snapshotSegment);
             }
             return snapshotSegments;
         }, executorService);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
index 8f87f0d49a20c..5dc3dcc7cb9a7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
@@ -23,8 +23,8 @@
 import java.util.List;
 import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
-import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
+import org.apache.pulsar.broker.delayed.proto.DelayedIndex;
+import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 import org.testng.Assert;
 import org.testng.annotations.Test;
@@ -35,27 +35,19 @@ public class DelayedIndexQueueTest {
     @Test
     public void testCompare() {
         DelayedIndex delayedIndex =
-                DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L)
-                        .build();
+                new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L);
         DelayedIndex delayedIndex2 =
-                DelayedIndex.newBuilder().setTimestamp(2).setLedgerId(2L).setEntryId(2L)
-                        .build();
+                new DelayedIndex().setTimestamp(2).setLedgerId(2L).setEntryId(2L);
         Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0);
 
         delayedIndex =
-                DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L)
-                        .build();
+                new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L);
         delayedIndex2 =
-                DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(2L).setEntryId(2L)
-                        .build();
+                new DelayedIndex().setTimestamp(1).setLedgerId(2L).setEntryId(2L);
         Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0);
 
-        delayedIndex =
-                DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L)
-                        .build();
-        delayedIndex2 =
-                DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(2L)
-                        .build();
+        delayedIndex = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L);
+        delayedIndex2 = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(2L);
         Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0);
     }
 
@@ -63,21 +55,19 @@ public void testCompare() {
     public void testCombinedSegmentDelayedIndexQueue() {
         List listA = new ArrayList<>();
         for (int i = 0; i < 10; i++) {
-            DelayedIndex delayedIndex =
-                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L)
-                            .build();
+            DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L);
             listA.add(delayedIndex);
         }
-        SnapshotSegment snapshotSegmentA1 = SnapshotSegment.newBuilder().addAllIndexes(listA).build();
+        SnapshotSegment snapshotSegmentA1 = new SnapshotSegment();
+        snapshotSegmentA1.addAllIndexes(listA);
 
         List listA2 = new ArrayList<>();
         for (int i = 10; i < 20; i++) {
-            DelayedIndex delayedIndex =
-                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L)
-                            .build();
+            DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L);
             listA2.add(delayedIndex);
         }
-        SnapshotSegment snapshotSegmentA2 = SnapshotSegment.newBuilder().addAllIndexes(listA2).build();
+        SnapshotSegment snapshotSegmentA2 = new SnapshotSegment();
+        snapshotSegmentA2.addAllIndexes(listA2);
 
         List segmentListA = new ArrayList<>();
         segmentListA.add(snapshotSegmentA1);
@@ -85,36 +75,32 @@ public void testCombinedSegmentDelayedIndexQueue() {
 
         List listB = new ArrayList<>();
         for (int i = 0; i < 9; i++) {
-            DelayedIndex delayedIndex =
-                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L)
-                            .build();
+            DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L);
 
-            DelayedIndex delayedIndex2 =
-                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L)
-                            .build();
+            DelayedIndex delayedIndex2 = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L);
             listB.add(delayedIndex);
             listB.add(delayedIndex2);
         }
 
-        SnapshotSegment snapshotSegmentB = SnapshotSegment.newBuilder().addAllIndexes(listB).build();
+        SnapshotSegment snapshotSegmentB = new SnapshotSegment();
+        snapshotSegmentB.addAllIndexes(listB);
         List segmentListB = new ArrayList<>();
         segmentListB.add(snapshotSegmentB);
-        segmentListB.add(SnapshotSegment.newBuilder().build());
+        segmentListB.add(new SnapshotSegment());
 
         List listC = new ArrayList<>();
         for (int i = 10; i < 30; i+=2) {
             DelayedIndex delayedIndex =
-                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L)
-                            .build();
+                    new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L);
 
             DelayedIndex delayedIndex2 =
-                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L)
-                            .build();
+                    new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L);
             listC.add(delayedIndex);
             listC.add(delayedIndex2);
         }
 
-        SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build();
+        SnapshotSegment snapshotSegmentC = new SnapshotSegment();
+        snapshotSegmentC.addAllIndexes(listC);
         List segmentListC = new ArrayList<>();
         segmentListC.add(snapshotSegmentC);
 
@@ -123,11 +109,14 @@ public void testCombinedSegmentDelayedIndexQueue() {
 
         int count = 0;
         while (!delayedIndexQueue.isEmpty()) {
-            DelayedIndex pop = delayedIndexQueue.pop();
+            DelayedIndex pop = new DelayedIndex();
+            delayedIndexQueue.popToObject(pop);
             log.info("{} , {}, {}", pop.getTimestamp(), pop.getLedgerId(), pop.getEntryId());
             count++;
             if (!delayedIndexQueue.isEmpty()) {
                 DelayedIndex peek = delayedIndexQueue.peek();
+                long timestamp = delayedIndexQueue.peekTimestamp();
+                Assert.assertEquals(timestamp, peek.getTimestamp());
                 Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0);
             }
         }
@@ -147,10 +136,13 @@ public void TripleLongPriorityDelayedIndexQueueTest() {
 
         int count = 0;
         while (!delayedIndexQueue.isEmpty()) {
-            DelayedIndex pop = delayedIndexQueue.pop();
+            DelayedIndex pop = new DelayedIndex();
+            delayedIndexQueue.popToObject(pop);
             count++;
             if (!delayedIndexQueue.isEmpty()) {
                 DelayedIndex peek = delayedIndexQueue.peek();
+                long timestamp = delayedIndexQueue.peekTimestamp();
+                Assert.assertEquals(timestamp, peek.getTimestamp());
                 Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0);
             }
         }

From fce6e737a7ff785859117b216cbb12f999ddeb94 Mon Sep 17 00:00:00 2001
From: congbo <39078850+congbobo184@users.noreply.github.com>
Date: Fri, 21 Apr 2023 09:57:06 +0800
Subject: [PATCH 220/404] [improve][schema] Add admin cli for testCompatibility
 (#19974)











### Motivation

1. add admin cli `testCompatibility`
2. add `admin.schemas().testCompatibility()` test
### Modifications

1. add admin cli `testCompatibility`
2. add `admin.schemas().testCompatibility()` test
### Verifying this change
add test for it
- [ ] Make sure that the change passes the CI checks.

*(Please pick either of the following options)*

This change is a trivial rework / code cleanup without any test coverage.

*(or)*

This change is already covered by existing tests, such as *(please describe tests)*.

*(or)*

This change added tests and can be verified as follows:

*(example:)*
  - *Added integration tests for end-to-end deployment with large payloads (10MB)*
  - *Extended integration test for recovery after broker failure*

### Does this pull request potentially affect one of the following parts:



*If the box was checked, please highlight the changes*

- [ ] Dependencies (add or upgrade a dependency)
- [ ] The public API
- [ ] The schema
- [ ] The default values of configurations
- [ ] The threading model
- [ ] The binary protocol
- [ ] The REST endpoints
- [x] The admin CLI options
- [ ] The metrics
- [ ] Anything that affects deployment

### Documentation



- [ ] `doc` 
- [ ] `doc-required` 
- [x] `doc-not-needed` 
- [ ] `doc-complete` 

### Matching PR in forked repository

PR in forked repository: 
https://github.com/congbobo184/pulsar/pull/14


---
 .../broker/admin/AdminApiSchemaTest.java      | 29 +++++++++++++++++++
 .../schema/IsCompatibilityResponse.java       |  1 -
 .../pulsar/admin/cli/PulsarAdminToolTest.java |  5 ++++
 .../apache/pulsar/admin/cli/CmdSchemas.java   | 17 +++++++++++
 4 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java
index b37114f180216..f67bd6fcfce5b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java
@@ -49,6 +49,8 @@
 import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy;
 import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse;
+import org.apache.pulsar.common.protocol.schema.PostSchemaPayload;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.schema.SchemaInfoWithVersion;
 import org.apache.pulsar.common.schema.SchemaType;
@@ -438,4 +440,31 @@ public void testGetSchemaCompatibilityStrategyWhenSetBrokerLevelAndSchemaAutoUpd
                 admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace),
                 SchemaCompatibilityStrategy.UNDEFINED));
     }
+
+    @Test
+    public void testCompatibility() throws Exception {
+        String topicName = schemaCompatibilityNamespace + "/testCompatibility";
+        try {
+            admin.schemas().getSchemaInfo(topicName);
+            fail();
+        } catch (PulsarAdminException.NotFoundException e) {
+            assertEquals(e.getMessage(), "Schema not found");
+        }
+        Map properties = new HashMap<>();
+        PostSchemaPayload postSchemaPayload = new PostSchemaPayload("STRING", "", properties);
+        admin.schemas().createSchema(topicName, postSchemaPayload);
+        IsCompatibilityResponse isCompatibilityResponse =
+                admin.schemas().testCompatibility(topicName, postSchemaPayload);
+
+        assertTrue(isCompatibilityResponse.isCompatibility());
+        assertEquals(isCompatibilityResponse.getSchemaCompatibilityStrategy(), SchemaCompatibilityStrategy.FULL.name());
+        postSchemaPayload = new PostSchemaPayload("INT8", "", properties);
+        try {
+            admin.schemas().testCompatibility(topicName, postSchemaPayload);
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof PulsarAdminException.ServerSideErrorException);
+            assertTrue(e.getMessage().contains("Incompatible schema: exists schema type STRING, new schema type INT8"));
+        }
+    }
 }
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java
index d5edd96fab563..aa8b54d1a77be 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java
@@ -33,5 +33,4 @@
 public class IsCompatibilityResponse {
     boolean isCompatibility;
     String schemaCompatibilityStrategy;
-
 }
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
index 2fe7b0236f0e8..a1d5d695c6398 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
@@ -2360,6 +2360,11 @@ void schemas() throws Exception {
         PostSchemaPayload input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class);
         verify(schemas).createSchema("persistent://tn1/ns1/tp1", input);
 
+        cmdSchemas = new CmdSchemas(() -> admin);
+        cmdSchemas.run(split("compatibility -f " + schemaFile + " persistent://tn1/ns1/tp1"));
+        input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class);
+        verify(schemas).testCompatibility("persistent://tn1/ns1/tp1", input);
+
         cmdSchemas = new CmdSchemas(() -> admin);
         String jarFile = PulsarAdminToolTest.class.getClassLoader()
                 .getResource("dummyexamples.jar").getFile();
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java
index 5383e39807b05..44ac143e3507e 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java
@@ -42,6 +42,7 @@ public CmdSchemas(Supplier admin) {
         jcommander.addCommand("delete", new DeleteSchema());
         jcommander.addCommand("upload", new UploadSchema());
         jcommander.addCommand("extract", new ExtractSchema());
+        jcommander.addCommand("compatibility", new TestCompatibility());
     }
 
     @Parameters(commandDescription = "Get the schema for a topic")
@@ -164,4 +165,20 @@ void run() throws Exception {
         }
     }
 
+    @Parameters(commandDescription = "Test schema compatibility")
+    private class TestCompatibility extends CliCommand {
+        @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+        private java.util.List params;
+
+        @Parameter(names = { "-f", "--filename" }, description = "filename", required = true)
+        private String schemaFileName;
+
+        @Override
+        void run() throws Exception {
+            String topic = validateTopicName(params);
+            PostSchemaPayload input = MAPPER.readValue(new File(schemaFileName), PostSchemaPayload.class);
+            getAdmin().schemas().testCompatibility(topic, input);
+        }
+    }
+
 }

From e5a833a2dcb7ce13ada4ca94714cc045a02de276 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Fri, 21 Apr 2023 16:06:22 +0800
Subject: [PATCH 221/404] [improve][broker] Move bitmap from lastMutableBucket
 to ImmutableBucket (#20156)

---
 .../bucket/BucketDelayedDeliveryTracker.java  |  3 +-
 .../broker/delayed/bucket/MutableBucket.java  | 41 +++++++++++++------
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index c90064c9137bb..67a7de1f01339 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -168,7 +168,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException {
         }
 
         try {
-            FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 2, TimeUnit.SECONDS);
+            FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 5, TimeUnit.SECONDS);
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
             log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e);
             if (e instanceof InterruptedException) {
@@ -343,6 +343,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
             // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range,
             // enter sharedBucketPriorityQueue directly
             sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId);
+            lastMutableBucket.putIndexBit(ledgerId, entryId);
         } else {
             checkArgument(ledgerId >= lastMutableBucket.endLedgerId);
             lastMutableBucket.addMessage(ledgerId, entryId, deliverAt);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index b7e9e68f1bdc7..1173a401a8903 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -19,7 +19,7 @@
 package org.apache.pulsar.broker.delayed.bucket;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import com.google.protobuf.ByteString;
+import com.google.protobuf.UnsafeByteOperations;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -74,6 +74,8 @@ Pair createImmutableBucketAndAsyncPersistent(
 
         List bucketSnapshotSegments = new ArrayList<>();
         List segmentMetadataList = new ArrayList<>();
+        Map immutableBucketBitMap = new HashMap<>();
+
         Map bitMap = new HashMap<>();
         SnapshotSegment snapshotSegment = new SnapshotSegment();
         SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder();
@@ -82,18 +84,20 @@ Pair createImmutableBucketAndAsyncPersistent(
         long currentTimestampUpperLimit = 0;
         long currentFirstTimestamp = 0L;
         while (!delayedIndexQueue.isEmpty()) {
-            DelayedIndex delayedIndex = snapshotSegment.addIndexe();
-            delayedIndexQueue.popToObject(delayedIndex);
-
-            long timestamp = delayedIndex.getTimestamp();
+            final long timestamp = delayedIndexQueue.peekTimestamp();
             if (currentTimestampUpperLimit == 0) {
                 currentFirstTimestamp = timestamp;
                 firstScheduleTimestamps.add(currentFirstTimestamp);
                 currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1;
             }
 
-            long ledgerId = delayedIndex.getLedgerId();
-            long entryId = delayedIndex.getEntryId();
+            DelayedIndex delayedIndex = snapshotSegment.addIndexe();
+            delayedIndexQueue.popToObject(delayedIndex);
+
+            final long ledgerId = delayedIndex.getLedgerId();
+            final long entryId = delayedIndex.getEntryId();
+
+            removeIndexBit(ledgerId, entryId);
 
             checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId);
 
@@ -102,10 +106,10 @@ Pair createImmutableBucketAndAsyncPersistent(
                 sharedQueue.add(timestamp, ledgerId, entryId);
             }
 
-            numMessages++;
-
             bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1);
 
+            numMessages++;
+
             if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit
                     || (maxIndexesPerBucketSnapshotSegment != -1
                     && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
@@ -119,9 +123,17 @@ Pair createImmutableBucketAndAsyncPersistent(
                     final var lId = entry.getKey();
                     final var bm = entry.getValue();
                     bm.runOptimize();
-                    final var array = new byte[bm.serializedSizeInBytes()];
-                    bm.serialize(ByteBuffer.wrap(array));
-                    segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array));
+                    ByteBuffer byteBuffer = ByteBuffer.allocate(bm.serializedSizeInBytes());
+                    bm.serialize(byteBuffer);
+                    byteBuffer.flip();
+                    segmentMetadataBuilder.putDelayedIndexBitMap(lId, UnsafeByteOperations.unsafeWrap(byteBuffer));
+                    immutableBucketBitMap.compute(lId, (__, bm0) -> {
+                        if (bm0 == null) {
+                            return bm;
+                        }
+                        bm0.or(bm);
+                        return bm0;
+                    });
                     iterator.remove();
                 }
 
@@ -133,6 +145,10 @@ Pair createImmutableBucketAndAsyncPersistent(
             }
         }
 
+        // optimize bm
+        immutableBucketBitMap.values().forEach(RoaringBitmap::runOptimize);
+        this.delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize);
+
         SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder()
                 .addAllMetadataList(segmentMetadataList)
                 .build();
@@ -145,6 +161,7 @@ Pair createImmutableBucketAndAsyncPersistent(
         bucket.setNumberBucketDelayedMessages(numMessages);
         bucket.setLastSegmentEntryId(lastSegmentEntryId);
         bucket.setFirstScheduleTimestamps(firstScheduleTimestamps);
+        bucket.setDelayedIndexBitMap(immutableBucketBitMap);
 
         // Skip first segment, because it has already been loaded
         List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size());

From 6036dcce8d48f8e1ae311153eeb625f51c5aa827 Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Sat, 22 Apr 2023 10:58:05 +0800
Subject: [PATCH 222/404] [fix][client] Fix breaking changes for the deprecated
 methods of TopicMessageIdImpl (#20163)

---
 .../apache/pulsar/client/impl/TopicMessageIdImpl.java    | 2 +-
 .../pulsar/client/impl/TopicMessageIdImplTest.java       | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
index 00fe12b62b194..3dc9b23e93e86 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
@@ -62,7 +62,7 @@ public String getTopicPartitionName() {
 
     @Deprecated
     public MessageId getInnerMessageId() {
-        return new MessageIdImpl(getLedgerId(), getEntryId(), getPartitionIndex());
+        return msgId;
     }
 
     @Override
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java
index daf49f0e7752e..1ddd47af91dff 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java
@@ -20,6 +20,7 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertSame;
 
 import org.testng.annotations.Test;
 
@@ -54,4 +55,12 @@ public void equalsTest() {
         assertNotEquals(topicMsgId1, topicMsgId2);
     }
 
+    @Test
+    public void testDeprecatedMethods() {
+        BatchMessageIdImpl msgId = new BatchMessageIdImpl(1, 2, 3, 4);
+        TopicMessageIdImpl topicMsgId = new TopicMessageIdImpl("topic-partition-0", "topic", msgId);
+        assertSame(topicMsgId.getInnerMessageId(), msgId);
+        assertEquals(topicMsgId.getTopicPartitionName(), topicMsgId.getOwnerTopic());
+        assertEquals(topicMsgId.getTopicName(), "topic");
+    }
 }

From c58da14968f27cf2a4813db01f387b61a5a03636 Mon Sep 17 00:00:00 2001
From: StevenLuMT 
Date: Sat, 22 Apr 2023 21:22:26 +0800
Subject: [PATCH 223/404] [fix][test] fix bug for
 testcase(PersistentTopicsTest#testExamineMessage) (#20154)

Co-authored-by: lushiji 
---
 .../org/apache/pulsar/broker/admin/PersistentTopicsTest.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java
index 6949fe931ca5a..f80d9863a26a6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java
@@ -1144,6 +1144,7 @@ public void testExamineMessage() throws Exception {
         // Check examine message not allowed on partitioned topic.
         try {
             admin.topics().examineMessage(topicName, "earliest", 1);
+            Assert.fail("fail to check examine message not allowed on partitioned topic");
         } catch (PulsarAdminException e) {
             Assert.assertEquals(e.getMessage(),
                     "Examine messages on a partitioned topic is not allowed, please try examine message on specific "

From 4190e40ab4ac448a2f94edd1c621dc1c67d9ee4b Mon Sep 17 00:00:00 2001
From: Matteo Merli 
Date: Sat, 22 Apr 2023 13:28:53 -0700
Subject: [PATCH 224/404] [improve] Allow to build and push multi-arch Docker
 images (#19432)

Co-authored-by: Lari Hotari 
Co-authored-by: Yong Zhang 
Co-authored-by: Zixuan Liu 
Co-authored-by: tison 
---
 .github/workflows/pulsar-ci.yaml              |  1 +
 build/build_java_test_image.sh                |  1 +
 build/pulsar_ci_tool.sh                       |  3 +-
 docker/pulsar-all/pom.xml                     | 42 ++++++++++++---
 docker/pulsar/pom.xml                         | 51 +++++++++++++++----
 pom.xml                                       |  8 ++-
 .../latest-version-image/pom.xml              |  6 +++
 7 files changed, 94 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index f7dbc755264d6..4d4f2902d8f27 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -798,6 +798,7 @@ jobs:
           # build docker image
           # include building of Pulsar SQL, Connectors, Offloaders and server distros
           mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
+          -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
           -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
           -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 
diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh
index 0747e6dacb82a..459bf26f98eff 100755
--- a/build/build_java_test_image.sh
+++ b/build/build_java_test_image.sh
@@ -27,5 +27,6 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]
   SQUASH_PARAM="-Ddocker.squash=true"
 fi
 mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \
+  -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
   -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \
   "$@" install
\ No newline at end of file
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
index 61199eda2c5d8..d946edd395789 100755
--- a/build/pulsar_ci_tool.sh
+++ b/build/pulsar_ci_tool.sh
@@ -46,7 +46,8 @@ function ci_print_thread_dumps() {
 
 # runs maven
 function _ci_mvn() {
-  mvn -B -ntp "$@"
+  mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
+        "$@"
 }
 
 # runs OWASP Dependency Check for all projects
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index c63ff6d656957..2edd6c776dcf5 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -68,10 +68,6 @@
   
     
       docker
-      
-        target/apache-pulsar-io-connectors-${project.version}-bin
-        target/pulsar-offloader-distribution-${project.version}-bin.tar.gz
-      
       
       
         
@@ -143,17 +139,25 @@
                 
                   
                     
-                      pulsar-all
+                      ${docker.organization}/pulsar-all
                       
                         ${project.basedir}
                         
                           latest
+                          ${project.version}
                         
+                        
+                          target/apache-pulsar-io-connectors-${project.version}-bin
+                          target/pulsar-offloader-distribution-${project.version}-bin.tar.gz
+                        
+                        
+                          
+                            ${docker.platforms}
+                          
+                        
                       
                     
                   
-                  latest
-                  ${docker.organization}
                 
               
             
@@ -161,5 +165,29 @@
         
       
     
+
+    
+      docker-push
+      
+        
+          
+            io.fabric8
+            docker-maven-plugin
+            
+              
+                default
+                package
+                
+                  build
+                  tag
+                  push
+                
+              
+            
+          
+        
+      
+    
+
   
 
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 647f68bf1672c..8c3b868315535 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -47,15 +47,14 @@
     
   
 
+  
+      mirror://mirrors.ubuntu.com/mirrors.txt
+      http://security.ubuntu.com/ubuntu/
+  
+
   
     
       docker
-      
-        target/pulsar-server-distribution-${project.version}-bin.tar.gz
-        ${pulsar.client.python.version}
-        ${env.UBUNTU_MIRROR}
-        ${env.UBUNTU_SECURITY_MIRROR}
-      
       
         
           
@@ -72,17 +71,27 @@
                 
                   
                     
-                      pulsar
+                      ${docker.organization}/pulsar
                       
+                        
+                          target/pulsar-server-distribution-${project.version}-bin.tar.gz
+                          ${pulsar.client.python.version}
+                          ${UBUNTU_MIRROR}
+                          ${UBUNTU_SECURITY_MIRROR}
+                        
                         ${project.basedir}
                         
                           latest
+                          ${project.version}
                         
+                        
+                          
+                            ${docker.platforms}
+                          
+                        
                       
                     
                   
-                  latest
-                  ${docker.organization}
                 
               
             
@@ -108,5 +117,29 @@
         
       
     
+
+    
+      docker-push
+      
+        
+          
+            io.fabric8
+            docker-maven-plugin
+            
+              
+                default
+                package
+                
+                  build
+                  tag
+                  push
+                
+              
+            
+          
+        
+      
+    
+
   
 
diff --git a/pom.xml b/pom.xml
index 22589a1719812..aef380c5cd09c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,6 +94,12 @@ flexible messaging model and an intuitive client API.
     UTF-8
     ${maven.build.timestamp}
     true
+
+    
+    
+
     
     
       --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
@@ -152,7 +158,7 @@ flexible messaging model and an intuitive client API.
     0.10.2
     1.6.2
     8.37
-    0.40.2
+    0.42.1
     true
     0.5.0
     3.19.6
diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml
index 8474f820c9566..c73a6f8a83970 100644
--- a/tests/docker-images/latest-version-image/pom.xml
+++ b/tests/docker-images/latest-version-image/pom.xml
@@ -147,6 +147,7 @@
                 package
                 
                   build
+                  tag
                 
                 
                   
@@ -159,6 +160,11 @@
                           ${project.version}
                         
                         true
+                        
+                          
+                            ${docker.platforms}
+                          
+                        
                       
                     
                   

From 0c50866fbc18525f82a04c3a918628b8b50a4de8 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Sun, 23 Apr 2023 09:41:28 +0800
Subject: [PATCH 225/404] [fix] [broker] Fast fix infinite HTTP call
 getSubscriptions caused by wrong topicName (#20131)

---
 .../admin/impl/PersistentTopicsBase.java      |  32 +++-
 ...rInfiniteHttpCallGetSubscriptionsTest.java | 143 ++++++++++++++++++
 2 files changed, 167 insertions(+), 8 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 7347d6dbf20a2..6b163ae04469b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -20,6 +20,7 @@
 
 import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign;
 import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName;
+import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.github.zafarkhaja.semver.Version;
@@ -1171,6 +1172,21 @@ protected CompletableFuture internalDeleteTopicAsync(boolean authoritative
                 .thenCompose(__ -> pulsar().getBrokerService().deleteTopic(topicName.toString(), force));
     }
 
+    /**
+     * There has a known bug will make Pulsar misidentifies "tp-partition-0-DLQ-partition-0" as "tp-partition-0-DLQ".
+     * You can see the details from PR https://github.com/apache/pulsar/pull/19841.
+     * This method is a quick fix and will be removed in master branch after #19841 and PIP 263 are done.
+     */
+    private boolean isUnexpectedTopicName(PartitionedTopicMetadata topicMetadata) {
+        if (!topicName.toString().contains(PARTITIONED_TOPIC_SUFFIX)){
+            return false;
+        }
+        if (topicMetadata.partitions <= 0){
+            return false;
+        }
+        return topicName.getPartition(0).toString().equals(topicName.toString());
+    }
+
     protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean authoritative) {
         CompletableFuture future;
         if (topicName.isGlobal()) {
@@ -1188,7 +1204,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut
                     } else {
                         getPartitionedTopicMetadataAsync(topicName, authoritative, false)
                                 .thenAccept(partitionMetadata -> {
-                            if (partitionMetadata.partitions > 0) {
+                            if (partitionMetadata.partitions > 0 && !isUnexpectedTopicName(partitionMetadata)) {
                                 try {
                                     final Set subscriptions =
                                             Collections.newSetFromMap(
@@ -3716,7 +3732,7 @@ protected CompletableFuture preValidation(boolean authoritative) {
                             .thenCompose(metadata -> {
                                 if (metadata.partitions > 0) {
                                     return validateTopicOwnershipAsync(TopicName.get(topicName.toString()
-                                    + TopicName.PARTITIONED_TOPIC_SUFFIX + 0), authoritative);
+                                    + PARTITIONED_TOPIC_SUFFIX + 0), authoritative);
                                 } else {
                                     return validateTopicOwnershipAsync(topicName, authoritative);
                                 }
@@ -4543,7 +4559,7 @@ protected CompletableFuture internalValidateClientVersionAsync() {
     private CompletableFuture validatePartitionTopicUpdateAsync(String topicName, int numberOfPartition) {
         return internalGetListAsync().thenCompose(existingTopicList -> {
             TopicName partitionTopicName = TopicName.get(domain(), namespaceName, topicName);
-            String prefix = partitionTopicName.getPartitionedTopicName() + TopicName.PARTITIONED_TOPIC_SUFFIX;
+            String prefix = partitionTopicName.getPartitionedTopicName() + PARTITIONED_TOPIC_SUFFIX;
             return getPartitionedTopicMetadataAsync(partitionTopicName, false, false)
                     .thenAccept(metadata -> {
                         int oldPartition = metadata.partitions;
@@ -4551,8 +4567,8 @@ private CompletableFuture validatePartitionTopicUpdateAsync(String topicNa
                             if (existingTopicName.startsWith(prefix)) {
                                 try {
                                     long suffix = Long.parseLong(existingTopicName.substring(
-                                            existingTopicName.indexOf(TopicName.PARTITIONED_TOPIC_SUFFIX)
-                                                    + TopicName.PARTITIONED_TOPIC_SUFFIX.length()));
+                                            existingTopicName.indexOf(PARTITIONED_TOPIC_SUFFIX)
+                                                    + PARTITIONED_TOPIC_SUFFIX.length()));
                                     // Skip partition of partitioned topic by making sure
                                     // the numeric suffix greater than old partition number.
                                     if (suffix >= oldPartition && suffix <= (long) numberOfPartition) {
@@ -4593,12 +4609,12 @@ private CompletableFuture validatePartitionTopicUpdateAsync(String topicNa
      */
     private CompletableFuture validateNonPartitionTopicNameAsync(String topicName) {
         CompletableFuture ret = CompletableFuture.completedFuture(null);
-        if (topicName.contains(TopicName.PARTITIONED_TOPIC_SUFFIX)) {
+        if (topicName.contains(PARTITIONED_TOPIC_SUFFIX)) {
             try {
                 // First check if what's after suffix "-partition-" is number or not, if not number then can create.
-                int partitionIndex = topicName.indexOf(TopicName.PARTITIONED_TOPIC_SUFFIX);
+                int partitionIndex = topicName.indexOf(PARTITIONED_TOPIC_SUFFIX);
                 long suffix = Long.parseLong(topicName.substring(partitionIndex
-                        + TopicName.PARTITIONED_TOPIC_SUFFIX.length()));
+                        + PARTITIONED_TOPIC_SUFFIX.length()));
                 TopicName partitionTopicName = TopicName.get(domain(),
                         namespaceName, topicName.substring(0, partitionIndex));
                 ret = getPartitionedTopicMetadataAsync(partitionTopicName, false, false)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java
new file mode 100644
index 0000000000000..2efc4f4e7803f
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.TopicType;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class TopicNameForInfiniteHttpCallGetSubscriptionsTest extends ProducerConsumerBase {
+
+    @BeforeMethod(alwaysRun = true)
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+    }
+
+    @Override
+    protected void doInitConf() throws Exception {
+        super.doInitConf();
+        conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED);
+        conf.setDefaultNumPartitions(1);
+    }
+
+    @AfterMethod(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @Test
+    public void testInfiniteHttpCallGetSubscriptions() throws Exception {
+        final String randomStr = UUID.randomUUID().toString().replaceAll("-", "");
+        final String partitionedTopicName = "persistent://my-property/my-ns/tp1_" + randomStr;
+        final String topic_p0 = partitionedTopicName + TopicName.PARTITIONED_TOPIC_SUFFIX + "0";
+        final String subscriptionName = "sub1";
+        final String topicDLQ = topic_p0 + "-" + subscriptionName + "-DLQ";
+
+        admin.topics().createPartitionedTopic(partitionedTopicName, 2);
+
+        // Do test.
+        ProducerAndConsumerEntry pcEntry = triggerDLQCreated(topic_p0, topicDLQ, subscriptionName);
+        admin.topics().getSubscriptions(topicDLQ);
+
+        // cleanup.
+        pcEntry.consumer.close();
+        pcEntry.producer.close();
+        admin.topics().deletePartitionedTopic(partitionedTopicName);
+    }
+
+    @Test
+    public void testInfiniteHttpCallGetSubscriptions2() throws Exception {
+        final String randomStr = UUID.randomUUID().toString().replaceAll("-", "");
+        final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0-abc";
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .create();
+
+        // Do test.
+        admin.topics().getSubscriptions(topicName);
+
+        // cleanup.
+        producer.close();
+    }
+
+    @Test
+    public void testInfiniteHttpCallGetSubscriptions3() throws Exception {
+        final String randomStr = UUID.randomUUID().toString().replaceAll("-", "");
+        final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0";
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .create();
+
+        // Do test.
+        admin.topics().getSubscriptions(topicName);
+
+        // cleanup.
+        producer.close();
+    }
+
+    @AllArgsConstructor
+    private static class ProducerAndConsumerEntry {
+        private Producer producer;
+        private Consumer consumer;
+    }
+
+    private ProducerAndConsumerEntry triggerDLQCreated(String topicName, String DLQName, String subscriptionName) throws Exception {
+        Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName(subscriptionName)
+                .subscriptionType(SubscriptionType.Shared)
+                .enableRetry(true)
+                .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(DLQName).maxRedeliverCount(2).build())
+                .receiverQueueSize(100)
+                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+                .subscribe();
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .create();
+        // send messages.
+        for (int i = 0; i < 5; i++) {
+            producer.newMessage()
+                    .value("value-" + i)
+                    .sendAsync();
+        }
+        producer.flush();
+        // trigger the DLQ created.
+        for (int i = 0; i < 20; i++) {
+            Message msg = consumer.receive(5, TimeUnit.SECONDS);
+            if (msg != null) {
+                consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS);
+            } else {
+                break;
+            }
+        }
+
+        return new ProducerAndConsumerEntry(producer, consumer);
+    }
+}
\ No newline at end of file

From d3158bfebf9565ea9422a9116749628648e9f90d Mon Sep 17 00:00:00 2001
From: WangJialing <65590138+wangjialing218@users.noreply.github.com>
Date: Sun, 23 Apr 2023 10:07:02 +0800
Subject: [PATCH 226/404] [fix][broker] Fix getPartitionedStats miss
 subscription's messageAckRate (#19870)

---
 .../common/policies/data/stats/SubscriptionStatsImpl.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index 25fa666523f3c..7ae439829b319 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -160,6 +160,8 @@ public void reset() {
         bytesOutCounter = 0;
         msgOutCounter = 0;
         msgRateRedeliver = 0;
+        messageAckRate = 0;
+        chunkedMessageRate = 0;
         msgBacklog = 0;
         backlogSize = 0;
         msgBacklogNoDelayed = 0;
@@ -190,6 +192,8 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) {
         this.bytesOutCounter += stats.bytesOutCounter;
         this.msgOutCounter += stats.msgOutCounter;
         this.msgRateRedeliver += stats.msgRateRedeliver;
+        this.messageAckRate += stats.messageAckRate;
+        this.chunkedMessageRate += stats.chunkedMessageRate;
         this.msgBacklog += stats.msgBacklog;
         this.backlogSize += stats.backlogSize;
         this.msgBacklogNoDelayed += stats.msgBacklogNoDelayed;

From 963260abfa142b8cf9ffe372d85d470a78c17235 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Sun, 23 Apr 2023 11:27:48 +0800
Subject: [PATCH 227/404] [fix] [broker] Make `LeastResourceUsageWithWeight`
 thread safe (#20159)

Fix #20160

### Motivation
LeastResourceUsageWithWeight is an stateful object and current code will be accessed by multithread.
thread 1: is execute https://github.com/apache/pulsar/blob/2b41e4eafb1cba0e548dd90df60e8cdbb24cd490/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java#L89-L91
and thread 2: is execute https://github.com/apache/pulsar/blob/2b41e4eafb1cba0e548dd90df60e8cdbb24cd490/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java#L147-L150
so an IndexOutOfBound occurs.

### Modifications

change the state to `ThreadLocal`
---
 .../strategy/LeastResourceUsageWithWeight.java | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
index 902cfdaf73f5f..98986d84b9858 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
@@ -23,6 +23,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.concurrent.ThreadSafe;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
@@ -35,14 +36,15 @@
  * cause cluster fluctuations due to short-term load jitter.
  */
 @Slf4j
+@ThreadSafe
 public class LeastResourceUsageWithWeight implements BrokerSelectionStrategy {
     // Maintain this list to reduce object creation.
-    private final ArrayList bestBrokers;
-    private final Set noLoadDataBrokers;
+    private final ThreadLocal> bestBrokers;
+    private final ThreadLocal> noLoadDataBrokers;
 
     public LeastResourceUsageWithWeight() {
-        this.bestBrokers = new ArrayList<>();
-        this.noLoadDataBrokers = new HashSet<>();
+        this.bestBrokers = ThreadLocal.withInitial(ArrayList::new);
+        this.noLoadDataBrokers = ThreadLocal.withInitial(HashSet::new);
     }
 
     // A broker's max resource usage with weight using its historical load and short-term load data with weight.
@@ -70,7 +72,6 @@ private double getMaxResourceUsageWithWeight(final String broker, final BrokerLo
 
     /**
      * Find a suitable broker to assign the given bundle to.
-     * This method is not thread safety.
      *
      * @param candidates     The candidates for which the bundle may be assigned.
      * @param bundleToAssign The data for the bundle to assign.
@@ -86,6 +87,9 @@ public Optional select(
             return Optional.empty();
         }
 
+        ArrayList bestBrokers = this.bestBrokers.get();
+        HashSet noLoadDataBrokers = this.noLoadDataBrokers.get();
+
         bestBrokers.clear();
         noLoadDataBrokers.clear();
         // Maintain of list of all the best scoring brokers and then randomly
@@ -135,9 +139,7 @@ public Optional select(
                 log.info("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, "
                         + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size());
             }
-            for (String broker : candidates) {
-                bestBrokers.add(broker);
-            }
+            bestBrokers.addAll(candidates);
         }
 
         if (debugMode) {

From 6e585acd7d788c0979c113a92e341364c987cbe4 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Sun, 23 Apr 2023 13:42:10 +0800
Subject: [PATCH 228/404] [fix] [broker] Fix wrong logic of method
 TopicName.getPartition(int index) (#19841)

---
 .../PartitionKeywordCompatibilityTest.java    |   3 +-
 ...rInfiniteHttpCallGetSubscriptionsTest.java | 143 ------------------
 .../pulsar/common/naming/TopicName.java       |   2 +-
 .../pulsar/common/naming/TopicNameTest.java   |   8 +
 4 files changed, 11 insertions(+), 145 deletions(-)
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java
index fdf2eb29c5e9e..86a5fcdc05aec 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java
@@ -26,6 +26,7 @@
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
@@ -66,7 +67,7 @@ public void testAutoCreatePartitionTopicWithKeywordAndDeleteIt()
                 .subscribe();
         List topics = admin.topics().getList("public/default");
         List partitionedTopicList = admin.topics().getPartitionedTopicList("public/default");
-        Assert.assertTrue(topics.contains(topicName));
+        Assert.assertTrue(topics.contains(TopicName.get(topicName).getPartition(0).toString()));
         Assert.assertTrue(partitionedTopicList.contains(topicName));
         consumer.close();
         admin.topics().deletePartitionedTopic(topicName);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java
deleted file mode 100644
index 2efc4f4e7803f..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.api;
-
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import lombok.AllArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.TopicType;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-@Slf4j
-@Test(groups = "broker")
-public class TopicNameForInfiniteHttpCallGetSubscriptionsTest extends ProducerConsumerBase {
-
-    @BeforeMethod(alwaysRun = true)
-    @Override
-    protected void setup() throws Exception {
-        super.internalSetup();
-        super.producerBaseSetup();
-    }
-
-    @Override
-    protected void doInitConf() throws Exception {
-        super.doInitConf();
-        conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED);
-        conf.setDefaultNumPartitions(1);
-    }
-
-    @AfterMethod(alwaysRun = true)
-    @Override
-    protected void cleanup() throws Exception {
-        super.internalCleanup();
-    }
-
-    @Test
-    public void testInfiniteHttpCallGetSubscriptions() throws Exception {
-        final String randomStr = UUID.randomUUID().toString().replaceAll("-", "");
-        final String partitionedTopicName = "persistent://my-property/my-ns/tp1_" + randomStr;
-        final String topic_p0 = partitionedTopicName + TopicName.PARTITIONED_TOPIC_SUFFIX + "0";
-        final String subscriptionName = "sub1";
-        final String topicDLQ = topic_p0 + "-" + subscriptionName + "-DLQ";
-
-        admin.topics().createPartitionedTopic(partitionedTopicName, 2);
-
-        // Do test.
-        ProducerAndConsumerEntry pcEntry = triggerDLQCreated(topic_p0, topicDLQ, subscriptionName);
-        admin.topics().getSubscriptions(topicDLQ);
-
-        // cleanup.
-        pcEntry.consumer.close();
-        pcEntry.producer.close();
-        admin.topics().deletePartitionedTopic(partitionedTopicName);
-    }
-
-    @Test
-    public void testInfiniteHttpCallGetSubscriptions2() throws Exception {
-        final String randomStr = UUID.randomUUID().toString().replaceAll("-", "");
-        final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0-abc";
-        Producer producer = pulsarClient.newProducer(Schema.STRING)
-                .topic(topicName)
-                .create();
-
-        // Do test.
-        admin.topics().getSubscriptions(topicName);
-
-        // cleanup.
-        producer.close();
-    }
-
-    @Test
-    public void testInfiniteHttpCallGetSubscriptions3() throws Exception {
-        final String randomStr = UUID.randomUUID().toString().replaceAll("-", "");
-        final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0";
-        Producer producer = pulsarClient.newProducer(Schema.STRING)
-                .topic(topicName)
-                .create();
-
-        // Do test.
-        admin.topics().getSubscriptions(topicName);
-
-        // cleanup.
-        producer.close();
-    }
-
-    @AllArgsConstructor
-    private static class ProducerAndConsumerEntry {
-        private Producer producer;
-        private Consumer consumer;
-    }
-
-    private ProducerAndConsumerEntry triggerDLQCreated(String topicName, String DLQName, String subscriptionName) throws Exception {
-        Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
-                .topic(topicName)
-                .subscriptionName(subscriptionName)
-                .subscriptionType(SubscriptionType.Shared)
-                .enableRetry(true)
-                .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(DLQName).maxRedeliverCount(2).build())
-                .receiverQueueSize(100)
-                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
-                .subscribe();
-        Producer producer = pulsarClient.newProducer(Schema.STRING)
-                .topic(topicName)
-                .create();
-        // send messages.
-        for (int i = 0; i < 5; i++) {
-            producer.newMessage()
-                    .value("value-" + i)
-                    .sendAsync();
-        }
-        producer.flush();
-        // trigger the DLQ created.
-        for (int i = 0; i < 20; i++) {
-            Message msg = consumer.receive(5, TimeUnit.SECONDS);
-            if (msg != null) {
-                consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS);
-            } else {
-                break;
-            }
-        }
-
-        return new ProducerAndConsumerEntry(producer, consumer);
-    }
-}
\ No newline at end of file
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
index bab9c443ceaeb..79ef64c1ae459 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
@@ -231,7 +231,7 @@ public String getEncodedLocalName() {
     }
 
     public TopicName getPartition(int index) {
-        if (index == -1 || this.toString().contains(PARTITIONED_TOPIC_SUFFIX)) {
+        if (index == -1 || this.toString().endsWith(PARTITIONED_TOPIC_SUFFIX + index)) {
             return this;
         }
         String partitionName = this.toString() + PARTITIONED_TOPIC_SUFFIX + index;
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
index 0e4533e7a08d0..8e32fbe3d33c0 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
@@ -289,4 +289,12 @@ public void testShortTopicName() throws Exception {
             // Ok
         }
     }
+
+    @Test
+    public void testTwoKeyWordPartition(){
+        TopicName tp1 = TopicName.get("tenant1/namespace1/tp1-partition-0-DLQ");
+        TopicName tp2 = tp1.getPartition(0);
+        assertNotEquals(tp2.toString(), tp1.toString());
+        assertEquals(tp2.toString(), "persistent://tenant1/namespace1/tp1-partition-0-DLQ-partition-0");
+    }
 }

From d3d36bdbd22a1b1fa6cee41a91acd9ec8a4c9341 Mon Sep 17 00:00:00 2001
From: tiny-rain 
Date: Sun, 23 Apr 2023 13:45:03 +0800
Subject: [PATCH 229/404] [fix][meta] deadlock of zkSessionWatcher when
 zkConnection loss (#20122)

---
 .../org/apache/pulsar/metadata/impl/ZKSessionWatcher.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java
index 1ce01f57d4fbe..a840721023080 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java
@@ -81,7 +81,11 @@ public void close() throws Exception {
     }
 
     // task that runs every TICK_TIME to check zk connection
-    private synchronized void checkConnectionStatus() {
+    // NOT ThreadSafe:
+    // If zk client can't ensure the order, it may lead to problems.
+    // Currently,we only use it in single thread, it will be fine. but we shouldn't leave any potential problems
+    // in the future.
+    private void checkConnectionStatus() {
         try {
             CompletableFuture future = new CompletableFuture<>();
             zk.exists("/", false, (StatCallback) (rc, path, ctx, stat) -> {
@@ -126,7 +130,7 @@ synchronized void setSessionInvalid() {
         currentStatus = SessionEvent.SessionLost;
     }
 
-    private void checkState(Watcher.Event.KeeperState zkClientState) {
+    private synchronized void checkState(Watcher.Event.KeeperState zkClientState) {
         switch (zkClientState) {
         case Expired:
             if (currentStatus != SessionEvent.SessionLost) {

From 795d60acc7a27a2d6bddfc60ffd7c9cf52a91cb4 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Sun, 23 Apr 2023 16:15:35 +0800
Subject: [PATCH 230/404] [improve][cli] Add `--cleanupSubscription` to
 pulsar-admin (#20028)

Co-authored-by: tison 
---
 .../java/org/apache/pulsar/admin/cli/CmdFunctions.java |  7 +++++++
 .../java/org/apache/pulsar/admin/cli/CmdSinks.java     |  8 ++++++++
 .../apache/pulsar/functions/utils/SinkConfigUtils.java | 10 ----------
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 05bab9c6f198b..91ec7183ff13c 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -201,6 +201,9 @@ abstract class FunctionDetailsCommand extends BaseCommand {
         protected String className;
         @Parameter(names = { "-t", "--function-type" }, description = "The built-in Pulsar Function type")
         protected String functionType;
+        @Parameter(names = "--cleanup-subscription", description = "Whether delete the subscription "
+                + "when function is deleted")
+        protected Boolean cleanupSubscription;
         @Parameter(names = "--jar", description = "Path to the JAR file for the function "
                 + "(if the function is written in Java). It also supports URL path [http/https/file "
                 + "(file protocol assumes that file already exists on worker host)/function "
@@ -471,6 +474,10 @@ void processArguments() throws Exception {
                 }
             }
 
+            if (null != cleanupSubscription) {
+                functionConfig.setCleanupSubscription(cleanupSubscription);
+            }
+
             if (null != inputs) {
                 List inputTopics = Arrays.asList(inputs.split(","));
                 functionConfig.setInputs(inputTopics);
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
index 4af9221dd2eab..0b27dd8d0a737 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
@@ -291,6 +291,10 @@ abstract class SinkDetailsCommand extends BaseCommand {
         @Parameter(names = { "-t", "--sink-type" }, description = "The sinks's connector provider")
         protected String sinkType;
 
+        @Parameter(names = "--cleanup-subscription", description = "Whether delete the subscription "
+                + "when sink is deleted")
+        protected Boolean cleanupSubscription;
+
         @Parameter(names = { "-i",
                 "--inputs" }, description = "The sink's input topic or topics "
                 + "(multiple topics can be specified as a comma-separated list)")
@@ -469,6 +473,10 @@ void processArguments() throws Exception {
                 sinkConfig.setProcessingGuarantees(processingGuarantees);
             }
 
+            if (null != cleanupSubscription) {
+                sinkConfig.setCleanupSubscription(cleanupSubscription);
+            }
+
             if (retainOrdering != null) {
                 sinkConfig.setRetainOrdering(retainOrdering);
             }
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java
index d79f787588c95..1a4009f5295e2 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java
@@ -202,12 +202,6 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail
             sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(sinkConfig.getNegativeAckRedeliveryDelayMs());
         }
 
-        if (sinkConfig.getCleanupSubscription() != null) {
-            sourceSpecBuilder.setCleanupSubscription(sinkConfig.getCleanupSubscription());
-        } else {
-            sourceSpecBuilder.setCleanupSubscription(true);
-        }
-
         if (sinkConfig.getSourceSubscriptionPosition() == SubscriptionInitialPosition.Earliest) {
             sourceSpecBuilder.setSubscriptionPosition(Function.SubscriptionPosition.EARLIEST);
         } else {
@@ -329,7 +323,6 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) {
         // Set subscription position
         sinkConfig.setSourceSubscriptionPosition(
                 convertFromFunctionDetailsSubscriptionPosition(functionDetails.getSource().getSubscriptionPosition()));
-        sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription());
 
         if (functionDetails.getSource().getTimeoutMs() != 0) {
             sinkConfig.setTimeoutMs(functionDetails.getSource().getTimeoutMs());
@@ -671,9 +664,6 @@ public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig ne
         if (!StringUtils.isEmpty(newConfig.getCustomRuntimeOptions())) {
             mergedConfig.setCustomRuntimeOptions(newConfig.getCustomRuntimeOptions());
         }
-        if (newConfig.getCleanupSubscription() != null) {
-            mergedConfig.setCleanupSubscription(newConfig.getCleanupSubscription());
-        }
         if (newConfig.getTransformFunction() != null) {
             mergedConfig.setTransformFunction(newConfig.getTransformFunction());
         }

From ab8a8c9ef58c0d3b7c0838df81f9762637c24f90 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Mon, 24 Apr 2023 00:26:23 +0800
Subject: [PATCH 231/404] [fix][broker] Remove useless field in the
 DelayedMessageIndexBucketSegment.proto (#20166)

---
 .../src/main/proto/DelayedMessageIndexBucketSegment.proto        | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto
index 633d6a8f1615c..a6ed30cfe8cd4 100644
--- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto
+++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto
@@ -31,5 +31,4 @@ message DelayedIndex {
 
 message SnapshotSegment {
     repeated DelayedIndex indexes = 1;
-    map delayed_index_bit_map = 2;
 }

From 35e9897742b7db4bd29349940075a819b2ad6999 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Mon, 24 Apr 2023 10:34:20 +0800
Subject: [PATCH 232/404] [fix][broker] Release EntryBuffer after parse proto
 object (#20170)

---
 .../BookkeeperBucketSnapshotStorage.java      | 30 +++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 040bbbc586f49..e99f39b382f56 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -20,6 +20,7 @@
 
 import com.google.protobuf.InvalidProtocolBufferException;
 import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
@@ -38,6 +39,7 @@
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata;
 import org.apache.pulsar.broker.delayed.proto.SnapshotSegment;
+import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.util.FutureUtil;
 
 @Slf4j
@@ -60,8 +62,9 @@ public BookkeeperBucketSnapshotStorage(PulsarService pulsar) {
     public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata,
                                                         List bucketSnapshotSegments,
                                                         String bucketKey, String topicName, String cursorName) {
+        ByteBuf metadataByteBuf = Unpooled.wrappedBuffer(snapshotMetadata.toByteArray());
         return createLedger(bucketKey, topicName, cursorName)
-                .thenCompose(ledgerHandle -> addEntry(ledgerHandle, snapshotMetadata.toByteArray())
+                .thenCompose(ledgerHandle -> addEntry(ledgerHandle, metadataByteBuf)
                         .thenCompose(__ -> addSnapshotSegments(ledgerHandle, bucketSnapshotSegments))
                         .thenCompose(__ -> closeLedger(ledgerHandle))
                         .thenApply(__ -> ledgerHandle.getId()));
@@ -117,19 +120,32 @@ public void close() throws Exception {
     private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle,
                                                         List bucketSnapshotSegments) {
         List> addFutures = new ArrayList<>();
+        ByteBuf byteBuf;
         for (SnapshotSegment bucketSnapshotSegment : bucketSnapshotSegments) {
-            addFutures.add(addEntry(ledgerHandle, bucketSnapshotSegment.toByteArray()));
+            byteBuf = PulsarByteBufAllocator.DEFAULT.directBuffer(bucketSnapshotSegment.getSerializedSize());
+            try {
+                bucketSnapshotSegment.writeTo(byteBuf);
+            } catch (Exception e){
+                byteBuf.release();
+                throw e;
+            }
+            addFutures.add(addEntry(ledgerHandle, byteBuf));
         }
 
         return FutureUtil.waitForAll(addFutures);
     }
 
     private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) {
+        ByteBuf entryBuffer = null;
         try {
-            ByteBuf entryBuffer = ledgerEntry.getEntryBuffer();
+            entryBuffer = ledgerEntry.getEntryBuffer();
             return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer());
         } catch (InvalidProtocolBufferException e) {
             throw new BucketSnapshotSerializationException(e);
+        } finally {
+            if (entryBuffer != null) {
+                entryBuffer.release();
+            }
         }
     }
 
@@ -139,7 +155,11 @@ private List parseSnapshotSegmentEntries(Enumeration closeLedger(LedgerHandle ledgerHandle) {
         return future;
     }
 
-    private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data) {
+    private CompletableFuture addEntry(LedgerHandle ledgerHandle, ByteBuf data) {
         final CompletableFuture future = new CompletableFuture<>();
         ledgerHandle.asyncAddEntry(data,
                 (rc, handle, entryId, ctx) -> {

From fd60e9e8380af1c7680999cbb5bff8160ba3571a Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Tue, 25 Apr 2023 17:22:41 +0800
Subject: [PATCH 233/404] [improve][build] Upgrade jackson version to 2.15.0
 for CVE-2022-1471 (#20177)

---
 .../server/src/assemble/LICENSE.bin.txt       | 22 +++++++--------
 .../shell/src/assemble/LICENSE.bin.txt        | 22 +++++++--------
 pom.xml                                       |  2 +-
 .../pulsar/common/util/FieldParser.java       |  7 ++---
 pulsar-sql/presto-distribution/LICENSE        | 28 +++++++++----------
 5 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 26651246ddf48..312c79dee1ab6 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -246,17 +246,17 @@ The Apache Software License, Version 2.0
  * JCommander -- com.beust-jcommander-1.82.jar
  * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar
  * Jackson
-     - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar
-     - com.fasterxml.jackson.core-jackson-core-2.14.2.jar
-     - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar
-     - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar
-     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar
-     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar
-     - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar
-     - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar
-     - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar
-     - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar
-     - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar
+     - com.fasterxml.jackson.core-jackson-annotations-2.15.0.jar
+     - com.fasterxml.jackson.core-jackson-core-2.15.0.jar
+     - com.fasterxml.jackson.core-jackson-databind-2.15.0.jar
+     - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.15.0.jar
+     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.15.0.jar
+     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.15.0.jar
+     - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.15.0.jar
+     - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.15.0.jar
+     - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.15.0.jar
+     - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.15.0.jar
+     - com.fasterxml.jackson.module-jackson-module-parameter-names-2.15.0.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
  * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 711890809f1bf..11ef4f3d4e745 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses:
 The Apache Software License, Version 2.0
  * JCommander -- jcommander-1.82.jar
  * Jackson
-     - jackson-annotations-2.14.2.jar
-     - jackson-core-2.14.2.jar
-     - jackson-databind-2.14.2.jar
-     - jackson-dataformat-yaml-2.14.2.jar
-     - jackson-jaxrs-base-2.14.2.jar
-     - jackson-jaxrs-json-provider-2.14.2.jar
-     - jackson-module-jaxb-annotations-2.14.2.jar
-     - jackson-module-jsonSchema-2.14.2.jar
-     - jackson-datatype-jdk8-2.14.2.jar
-     - jackson-datatype-jsr310-2.14.2.jar
-     - jackson-module-parameter-names-2.14.2.jar
+     - jackson-annotations-2.15.0.jar
+     - jackson-core-2.15.0.jar
+     - jackson-databind-2.15.0.jar
+     - jackson-dataformat-yaml-2.15.0.jar
+     - jackson-jaxrs-base-2.15.0.jar
+     - jackson-jaxrs-json-provider-2.15.0.jar
+     - jackson-module-jaxb-annotations-2.15.0.jar
+     - jackson-module-jsonSchema-2.15.0.jar
+     - jackson-datatype-jdk8-2.15.0.jar
+     - jackson-datatype-jsr310-2.15.0.jar
+     - jackson-module-parameter-names-2.15.0.jar
  * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar
  * Gson
     - gson-2.8.9.jar
diff --git a/pom.xml b/pom.xml
index aef380c5cd09c..28155b20cb4f5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,7 @@ flexible messaging model and an intuitive client API.
     1.69
     1.0.6
     1.0.2.3
-    2.14.2
+    2.15.0
     0.10.2
     1.6.2
     8.37
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java
index 626a14b92eedd..c1c17419abbcb 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java
@@ -21,8 +21,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
-import com.fasterxml.jackson.databind.AnnotationIntrospector;
-import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
 import com.fasterxml.jackson.databind.util.EnumResolver;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
@@ -58,8 +56,6 @@ public final class FieldParser {
     private static final Map CONVERTERS = new HashMap<>();
     private static final Map, Class> WRAPPER_TYPES = new HashMap<>();
 
-    private static final AnnotationIntrospector ANNOTATION_INTROSPECTOR = new JacksonAnnotationIntrospector();
-
     static {
         // Preload converters and wrapperTypes.
         initConverters();
@@ -100,7 +96,8 @@ public static  T convert(Object from, Class to) {
 
         if (to.isEnum()) {
             // Converting string to enum
-            EnumResolver r = EnumResolver.constructUsingToString((Class>) to, ANNOTATION_INTROSPECTOR);
+            EnumResolver r = EnumResolver.constructUsingToString(
+                    ObjectMapperFactory.getMapper().getObjectMapper().getDeserializationConfig(), to);
             T value = (T) r.findEnum((String) from);
             if (value == null) {
                 throw new RuntimeException("Invalid value '" + from + "' for enum " + to);
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 09d1396b70419..407cd9a0eb9e4 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses:
 The Apache Software License, Version 2.0
 
   * Jackson
-    - jackson-annotations-2.14.2.jar
-    - jackson-core-2.14.2.jar
-    - jackson-databind-2.14.2.jar
-    - jackson-dataformat-smile-2.14.2.jar
-    - jackson-datatype-guava-2.14.2.jar
-    - jackson-datatype-jdk8-2.14.2.jar
-    - jackson-datatype-joda-2.14.2.jar
-    - jackson-datatype-jsr310-2.14.2.jar
-    - jackson-dataformat-yaml-2.14.2.jar
-    - jackson-jaxrs-base-2.14.2.jar
-    - jackson-jaxrs-json-provider-2.14.2.jar
-    - jackson-module-jaxb-annotations-2.14.2.jar
-    - jackson-module-jsonSchema-2.14.2.jar
+    - jackson-annotations-2.15.0.jar
+    - jackson-core-2.15.0.jar
+    - jackson-databind-2.15.0.jar
+    - jackson-dataformat-smile-2.15.0.jar
+    - jackson-datatype-guava-2.15.0.jar
+    - jackson-datatype-jdk8-2.15.0.jar
+    - jackson-datatype-joda-2.15.0.jar
+    - jackson-datatype-jsr310-2.15.0.jar
+    - jackson-dataformat-yaml-2.15.0.jar
+    - jackson-jaxrs-base-2.15.0.jar
+    - jackson-jaxrs-json-provider-2.15.0.jar
+    - jackson-module-jaxb-annotations-2.15.0.jar
+    - jackson-module-jsonSchema-2.15.0.jar
  * Guava
     - guava-31.0.1-jre.jar
     - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
@@ -456,7 +456,7 @@ The Apache Software License, Version 2.0
   * Snappy
     - snappy-java-1.1.8.4.jar
   * Jackson
-    - jackson-module-parameter-names-2.14.2.jar
+    - jackson-module-parameter-names-2.15.0.jar
   * Java Assist
     - javassist-3.25.0-GA.jar
   * Java Native Access

From 7bd9998c9974ff17b1fbd5b3db7cf5bd7d3bc518 Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Wed, 26 Apr 2023 10:49:30 +0800
Subject: [PATCH 234/404] [improve][build] Revert "Upgrade jackson version to
 2.15.0 for CVE-2022-1471 (#20177)" (#20182)

---
 .../server/src/assemble/LICENSE.bin.txt       | 22 +++++++--------
 .../shell/src/assemble/LICENSE.bin.txt        | 22 +++++++--------
 pom.xml                                       |  2 +-
 .../pulsar/common/util/FieldParser.java       |  7 +++--
 pulsar-sql/presto-distribution/LICENSE        | 28 +++++++++----------
 5 files changed, 42 insertions(+), 39 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 312c79dee1ab6..26651246ddf48 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -246,17 +246,17 @@ The Apache Software License, Version 2.0
  * JCommander -- com.beust-jcommander-1.82.jar
  * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar
  * Jackson
-     - com.fasterxml.jackson.core-jackson-annotations-2.15.0.jar
-     - com.fasterxml.jackson.core-jackson-core-2.15.0.jar
-     - com.fasterxml.jackson.core-jackson-databind-2.15.0.jar
-     - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.15.0.jar
-     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.15.0.jar
-     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.15.0.jar
-     - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.15.0.jar
-     - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.15.0.jar
-     - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.15.0.jar
-     - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.15.0.jar
-     - com.fasterxml.jackson.module-jackson-module-parameter-names-2.15.0.jar
+     - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar
+     - com.fasterxml.jackson.core-jackson-core-2.14.2.jar
+     - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar
+     - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar
+     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar
+     - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar
+     - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar
+     - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar
+     - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar
+     - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar
+     - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
  * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 11ef4f3d4e745..711890809f1bf 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses:
 The Apache Software License, Version 2.0
  * JCommander -- jcommander-1.82.jar
  * Jackson
-     - jackson-annotations-2.15.0.jar
-     - jackson-core-2.15.0.jar
-     - jackson-databind-2.15.0.jar
-     - jackson-dataformat-yaml-2.15.0.jar
-     - jackson-jaxrs-base-2.15.0.jar
-     - jackson-jaxrs-json-provider-2.15.0.jar
-     - jackson-module-jaxb-annotations-2.15.0.jar
-     - jackson-module-jsonSchema-2.15.0.jar
-     - jackson-datatype-jdk8-2.15.0.jar
-     - jackson-datatype-jsr310-2.15.0.jar
-     - jackson-module-parameter-names-2.15.0.jar
+     - jackson-annotations-2.14.2.jar
+     - jackson-core-2.14.2.jar
+     - jackson-databind-2.14.2.jar
+     - jackson-dataformat-yaml-2.14.2.jar
+     - jackson-jaxrs-base-2.14.2.jar
+     - jackson-jaxrs-json-provider-2.14.2.jar
+     - jackson-module-jaxb-annotations-2.14.2.jar
+     - jackson-module-jsonSchema-2.14.2.jar
+     - jackson-datatype-jdk8-2.14.2.jar
+     - jackson-datatype-jsr310-2.14.2.jar
+     - jackson-module-parameter-names-2.14.2.jar
  * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar
  * Gson
     - gson-2.8.9.jar
diff --git a/pom.xml b/pom.xml
index 28155b20cb4f5..aef380c5cd09c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,7 @@ flexible messaging model and an intuitive client API.
     1.69
     1.0.6
     1.0.2.3
-    2.15.0
+    2.14.2
     0.10.2
     1.6.2
     8.37
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java
index c1c17419abbcb..626a14b92eedd 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java
@@ -21,6 +21,8 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
+import com.fasterxml.jackson.databind.AnnotationIntrospector;
+import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
 import com.fasterxml.jackson.databind.util.EnumResolver;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
@@ -56,6 +58,8 @@ public final class FieldParser {
     private static final Map CONVERTERS = new HashMap<>();
     private static final Map, Class> WRAPPER_TYPES = new HashMap<>();
 
+    private static final AnnotationIntrospector ANNOTATION_INTROSPECTOR = new JacksonAnnotationIntrospector();
+
     static {
         // Preload converters and wrapperTypes.
         initConverters();
@@ -96,8 +100,7 @@ public static  T convert(Object from, Class to) {
 
         if (to.isEnum()) {
             // Converting string to enum
-            EnumResolver r = EnumResolver.constructUsingToString(
-                    ObjectMapperFactory.getMapper().getObjectMapper().getDeserializationConfig(), to);
+            EnumResolver r = EnumResolver.constructUsingToString((Class>) to, ANNOTATION_INTROSPECTOR);
             T value = (T) r.findEnum((String) from);
             if (value == null) {
                 throw new RuntimeException("Invalid value '" + from + "' for enum " + to);
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 407cd9a0eb9e4..09d1396b70419 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses:
 The Apache Software License, Version 2.0
 
   * Jackson
-    - jackson-annotations-2.15.0.jar
-    - jackson-core-2.15.0.jar
-    - jackson-databind-2.15.0.jar
-    - jackson-dataformat-smile-2.15.0.jar
-    - jackson-datatype-guava-2.15.0.jar
-    - jackson-datatype-jdk8-2.15.0.jar
-    - jackson-datatype-joda-2.15.0.jar
-    - jackson-datatype-jsr310-2.15.0.jar
-    - jackson-dataformat-yaml-2.15.0.jar
-    - jackson-jaxrs-base-2.15.0.jar
-    - jackson-jaxrs-json-provider-2.15.0.jar
-    - jackson-module-jaxb-annotations-2.15.0.jar
-    - jackson-module-jsonSchema-2.15.0.jar
+    - jackson-annotations-2.14.2.jar
+    - jackson-core-2.14.2.jar
+    - jackson-databind-2.14.2.jar
+    - jackson-dataformat-smile-2.14.2.jar
+    - jackson-datatype-guava-2.14.2.jar
+    - jackson-datatype-jdk8-2.14.2.jar
+    - jackson-datatype-joda-2.14.2.jar
+    - jackson-datatype-jsr310-2.14.2.jar
+    - jackson-dataformat-yaml-2.14.2.jar
+    - jackson-jaxrs-base-2.14.2.jar
+    - jackson-jaxrs-json-provider-2.14.2.jar
+    - jackson-module-jaxb-annotations-2.14.2.jar
+    - jackson-module-jsonSchema-2.14.2.jar
  * Guava
     - guava-31.0.1-jre.jar
     - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
@@ -456,7 +456,7 @@ The Apache Software License, Version 2.0
   * Snappy
     - snappy-java-1.1.8.4.jar
   * Jackson
-    - jackson-module-parameter-names-2.15.0.jar
+    - jackson-module-parameter-names-2.14.2.jar
   * Java Assist
     - javassist-3.25.0-GA.jar
   * Java Native Access

From 0ec576b25fdd7e4890799184ac03f66deb4735f6 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Wed, 26 Apr 2023 22:16:39 +0800
Subject: [PATCH 235/404] [fix][test] Fix flaky test
 `ConcurrentBitmapSortedLongPairSetTest` (#20165)

---
 .../utils/ConcurrentBitmapSortedLongPairSetTest.java      | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java
index 47379c3b6f17c..5f8f13288cfe8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java
@@ -26,7 +26,6 @@
 import org.testng.annotations.Test;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -188,15 +187,12 @@ public void concurrentInsertions() throws Throwable {
         List> futures = new ArrayList<>();
         for (int i = 0; i < nThreads; i++) {
             final int threadIdx = i;
-
             futures.add(executor.submit(() -> {
-                Random random = new Random();
 
+                int start = N * (threadIdx + 1);
                 for (int j = 0; j < N; j++) {
-                    int key = random.nextInt();
+                    int key = start + j;
                     // Ensure keys are unique
-                    key -= key % (threadIdx + 1);
-                    key = Math.abs(key);
                     set.add(key, key);
                 }
             }));

From f7c0b3c49c9ad8c28d0b00aa30d727850eb8bc04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Wed, 26 Apr 2023 17:36:56 +0200
Subject: [PATCH 236/404] [improve][fn] Allow unknown fields in connectors
 config (#20116)

---
 conf/functions_worker.yml                     |   6 +
 .../functions/instance/InstanceConfig.java    |   1 +
 .../instance/JavaInstanceRunnable.java        |  97 ++++++++++++++--
 .../instance/JavaInstanceRunnableTest.java    | 104 +++++++++++++++---
 .../runtime/JavaInstanceStarter.java          |   7 ++
 .../functions/runtime/RuntimeUtils.java       |   6 +-
 .../pulsar/functions/worker/WorkerConfig.java |  11 ++
 .../functions/worker/FunctionActioner.java    |   1 +
 .../META-INF/services/pulsar-io.yaml          |   3 +-
 .../META-INF/services/pulsar-io.yaml          |   1 +
 .../META-INF/services/pulsar-io.yaml          |   1 +
 .../META-INF/services/pulsar-io.yaml          |   1 +
 .../META-INF/services/pulsar-io.yaml          |   1 +
 .../META-INF/services/pulsar-io.yaml          |   1 +
 14 files changed, 217 insertions(+), 24 deletions(-)

diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index bb15e0ca416da..4c5b6aab1b7f4 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -407,6 +407,12 @@ validateConnectorConfig: false
 # If it is set to true, you must ensure that it has been initialized by "bin/pulsar initialize-cluster-metadata" command.
 initializedDlogMetadata: false
 
+# Whether to ignore unknown properties when deserializing the connector configuration.
+# After upgrading a connector to a new version with a new configuration, the new configuration may not be compatible with the old connector.
+# In case of rollback, it's required to also rollback the connector configuration.
+# Ignoring unknown fields makes possible to keep the new configuration and only rollback the connector.
+ignoreUnknownConfigFields: false
+
 ###########################
 # Arbitrary Configuration
 ###########################
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
index 1a89505d9bb11..fcee6d734d6c9 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
@@ -48,6 +48,7 @@ public class InstanceConfig {
     private boolean exposePulsarAdminClientEnabled = false;
     private int metricsPort;
     private List additionalJavaRuntimeArguments = Collections.emptyList();
+    private boolean ignoreUnknownConfigFields;
 
     /**
      * Get the string representation of {@link #getInstanceId()}.
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
index e2ad9e4c989d1..c3f36f754daca 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
@@ -19,13 +19,20 @@
 package org.apache.pulsar.functions.instance;
 
 import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition;
+import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationConfig;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.deser.BeanDeserializer;
 import com.google.common.annotations.VisibleForTesting;
 import com.scurrilous.circe.checksum.Crc32cIntChecksum;
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
@@ -34,6 +41,7 @@
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Consumer;
 import lombok.Getter;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.typetools.TypeResolver;
 import org.apache.commons.lang3.StringUtils;
@@ -59,6 +67,7 @@
 import org.apache.pulsar.common.functions.ConsumerConfig;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.ProducerConfig;
+import org.apache.pulsar.common.nar.NarClassLoader;
 import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 import org.apache.pulsar.common.schema.KeyValue;
 import org.apache.pulsar.common.schema.KeyValueEncodingType;
@@ -94,6 +103,7 @@
 import org.apache.pulsar.functions.source.batch.BatchSourceExecutor;
 import org.apache.pulsar.functions.utils.CryptoUtils;
 import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.io.ConnectorUtils;
 import org.apache.pulsar.io.core.Sink;
 import org.apache.pulsar.io.core.Source;
 import org.slf4j.Logger;
@@ -855,10 +865,7 @@ private void setupInput(ContextImpl contextImpl) throws Exception {
             if (sourceSpec.getConfigs().isEmpty()) {
                 this.source.open(new HashMap<>(), contextImpl);
             } else {
-                this.source.open(
-                        ObjectMapperFactory.getMapper().reader().forType(new TypeReference>() {
-                        }).readValue(sourceSpec.getConfigs())
-                        , contextImpl);
+                this.source.open(parseComponentConfig(sourceSpec.getConfigs()), contextImpl);
             }
             if (this.source instanceof PulsarSource) {
                 contextImpl.setInputConsumers(((PulsarSource) this.source).getInputConsumers());
@@ -870,6 +877,83 @@ private void setupInput(ContextImpl contextImpl) throws Exception {
             Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
         }
     }
+    private Map parseComponentConfig(String connectorConfigs) throws IOException {
+        return parseComponentConfig(connectorConfigs, instanceConfig, componentClassLoader, componentType);
+    }
+
+    static Map parseComponentConfig(String connectorConfigs,
+                                                    InstanceConfig instanceConfig,
+                                                    ClassLoader componentClassLoader,
+                                                    org.apache.pulsar.functions.proto.Function
+                                                            .FunctionDetails.ComponentType componentType)
+            throws IOException {
+        final Map config = ObjectMapperFactory
+                .getMapper()
+                .reader()
+                .forType(new TypeReference>() {})
+                .readValue(connectorConfigs);
+        if (instanceConfig.isIgnoreUnknownConfigFields() && componentClassLoader instanceof NarClassLoader) {
+            final String configClassName;
+            if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) {
+                configClassName = ConnectorUtils
+                        .getConnectorDefinition((NarClassLoader) componentClassLoader).getSourceConfigClass();
+            } else if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
+                configClassName =  ConnectorUtils
+                        .getConnectorDefinition((NarClassLoader) componentClassLoader).getSinkConfigClass();
+            } else {
+                return config;
+            }
+            if (configClassName != null) {
+
+                Class configClass;
+                try {
+                    configClass = Class.forName(configClassName,
+                            true, Thread.currentThread().getContextClassLoader());
+                } catch (ClassNotFoundException e) {
+                    throw new RuntimeException("Config class not found: " + configClassName, e);
+                }
+                final List allFields = BeanPropertiesReader.getBeanProperties(configClass);
+
+                for (String s : config.keySet()) {
+                    if (!allFields.contains(s)) {
+                        log.error("Field '{}' not defined in the {} configuration {}, the field will be ignored",
+                                s,
+                                componentType,
+                                configClass);
+                        config.remove(s);
+                    }
+                }
+            }
+        }
+        return config;
+    }
+
+    static final class BeanPropertiesReader {
+
+        private static final MapperBeanReader reader = new MapperBeanReader();
+
+        private static final class MapperBeanReader extends ObjectMapper {
+            @SneakyThrows
+            List getBeanProperties(Class valueType) {
+                final JsonParser parser = ObjectMapperFactory
+                        .getMapper()
+                        .getObjectMapper()
+                        .createParser("");
+                DeserializationConfig config = getDeserializationConfig();
+                DeserializationContext ctxt = createDeserializationContext(parser, config);
+                BeanDeserializer deser = (BeanDeserializer)
+                        _findRootDeserializer(ctxt, _typeFactory.constructType(valueType));
+                List list = new ArrayList<>();
+                deser.properties().forEachRemaining(p -> list.add(p.getName()));
+                return list;
+            }
+        }
+
+        static List getBeanProperties(Class valueType) {
+            return reader.getBeanProperties(valueType);
+        }
+    }
+
 
     private void setupOutput(ContextImpl contextImpl) throws Exception {
 
@@ -940,9 +1024,8 @@ private void setupOutput(ContextImpl contextImpl) throws Exception {
                     log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec,
                             contextImpl.toString());
                 }
-                this.sink.open(ObjectMapperFactory.getMapper().reader().forType(
-                        new TypeReference>() {
-                        }).readValue(sinkSpec.getConfigs()), contextImpl);
+                final Map config = parseComponentConfig(sinkSpec.getConfigs());
+                this.sink.open(config, contextImpl);
             }
         } catch (Exception e) {
             log.error("Sink open produced uncaught exception: ", e);
diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java
index a36a3ca62d178..5fea8bcc9fde9 100644
--- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java
+++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java
@@ -18,19 +18,27 @@
  */
 package org.apache.pulsar.functions.instance;
 
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
+
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
+import java.util.TreeSet;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.pulsar.client.api.ClientBuilder;
+import org.apache.pulsar.common.io.ConnectorDefinition;
+import org.apache.pulsar.common.nar.NarClassLoader;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.functions.api.Context;
 import org.apache.pulsar.functions.api.Function;
 import org.apache.pulsar.functions.api.Record;
@@ -38,11 +46,10 @@
 import org.apache.pulsar.functions.instance.stats.ComponentStatsManager;
 import org.apache.pulsar.functions.proto.Function.FunctionDetails;
 import org.apache.pulsar.functions.proto.Function.SinkSpec;
-import org.apache.pulsar.functions.proto.Function.SinkSpecOrBuilder;
-import org.apache.pulsar.functions.proto.Function.SourceSpecOrBuilder;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.jetbrains.annotations.NotNull;
 import org.testng.Assert;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class JavaInstanceRunnableTest {
@@ -159,7 +166,7 @@ public void testFunctionResultNull() throws Exception {
 
     @NotNull
     private JavaInstanceRunnable getJavaInstanceRunnable(boolean autoAck,
-            org.apache.pulsar.functions.proto.Function.ProcessingGuarantees processingGuarantees) throws Exception {
+                                                         org.apache.pulsar.functions.proto.Function.ProcessingGuarantees processingGuarantees) throws Exception {
         FunctionDetails functionDetails = FunctionDetails.newBuilder()
                 .setAutoAck(autoAck)
                 .setProcessingGuarantees(processingGuarantees).build();
@@ -184,23 +191,90 @@ public void testStatsManagerNull() throws Exception {
 
     @Test
     public void testSinkConfigParsingPreservesOriginalType() throws Exception {
-        SinkSpecOrBuilder sinkSpec = mock(SinkSpecOrBuilder.class);
-        when(sinkSpec.getConfigs()).thenReturn("{\"ttl\": 9223372036854775807}");
-        Map parsedConfig =
-                new ObjectMapper().readValue(sinkSpec.getConfigs(), new TypeReference>() {
-                });
+        final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig(
+                "{\"ttl\": 9223372036854775807}",
+                new InstanceConfig(),
+                null,
+                FunctionDetails.ComponentType.SINK
+        );
         Assert.assertEquals(parsedConfig.get("ttl").getClass(), Long.class);
         Assert.assertEquals(parsedConfig.get("ttl"), Long.MAX_VALUE);
     }
 
     @Test
     public void testSourceConfigParsingPreservesOriginalType() throws Exception {
-        SourceSpecOrBuilder sourceSpec = mock(SourceSpecOrBuilder.class);
-        when(sourceSpec.getConfigs()).thenReturn("{\"ttl\": 9223372036854775807}");
-        Map parsedConfig =
-                new ObjectMapper().readValue(sourceSpec.getConfigs(), new TypeReference>() {
-                });
+        final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig(
+                "{\"ttl\": 9223372036854775807}",
+                new InstanceConfig(),
+                null,
+                FunctionDetails.ComponentType.SOURCE
+        );
         Assert.assertEquals(parsedConfig.get("ttl").getClass(), Long.class);
         Assert.assertEquals(parsedConfig.get("ttl"), Long.MAX_VALUE);
     }
+
+
+    public static class ConnectorTestConfig1 {
+        public String field1;
+    }
+
+    @DataProvider(name = "configIgnoreUnknownFields")
+    public static Object[][] configIgnoreUnknownFields() {
+        return new Object[][]{
+                {false, FunctionDetails.ComponentType.SINK},
+                {true, FunctionDetails.ComponentType.SINK},
+                {false, FunctionDetails.ComponentType.SOURCE},
+                {true, FunctionDetails.ComponentType.SOURCE}
+        };
+    }
+
+    @Test(dataProvider = "configIgnoreUnknownFields")
+    public void testSinkConfigIgnoreUnknownFields(boolean ignoreUnknownConfigFields,
+                                                  FunctionDetails.ComponentType type) throws Exception {
+        NarClassLoader narClassLoader = mock(NarClassLoader.class);
+        final ConnectorDefinition connectorDefinition = new ConnectorDefinition();
+        if (type == FunctionDetails.ComponentType.SINK) {
+            connectorDefinition.setSinkConfigClass(ConnectorTestConfig1.class.getName());
+        } else {
+            connectorDefinition.setSourceConfigClass(ConnectorTestConfig1.class.getName());
+        }
+        when(narClassLoader.getServiceDefinition(any())).thenReturn(ObjectMapperFactory
+                .getMapper().writer().writeValueAsString(connectorDefinition));
+        final InstanceConfig instanceConfig = new InstanceConfig();
+        instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields);
+
+        final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig(
+                "{\"field1\": \"value\", \"field2\": \"value2\"}",
+                instanceConfig,
+                narClassLoader,
+                type
+        );
+        if (ignoreUnknownConfigFields) {
+            Assert.assertEquals(parsedConfig.size(), 1);
+            Assert.assertEquals(parsedConfig.get("field1"), "value");
+        } else {
+            Assert.assertEquals(parsedConfig.size(), 2);
+            Assert.assertEquals(parsedConfig.get("field1"), "value");
+            Assert.assertEquals(parsedConfig.get("field2"), "value2");
+        }
+    }
+
+    public static class ConnectorTestConfig2 {
+        public static int constantField = 1;
+        public String field1;
+        private long withGetter;
+        @JsonIgnore
+        private ConnectorTestConfig1 ignore;
+
+        public long getWithGetter() {
+            return withGetter;
+        }
+    }
+
+    @Test
+    public void testBeanPropertiesReader() throws Exception {
+        final List beanProperties = JavaInstanceRunnable.BeanPropertiesReader
+                .getBeanProperties(ConnectorTestConfig2.class);
+        Assert.assertEquals(new TreeSet<>(beanProperties), new TreeSet<>(Arrays.asList("field1", "withGetter")));
+    }
 }
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index deff690815d9c..c4f44be3df380 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -150,6 +150,12 @@ public class JavaInstanceStarter implements AutoCloseable {
             + "exposed to function context, default is disabled.", required = false)
     public Boolean exposePulsarAdminClientEnabled = false;
 
+    @Parameter(names = "--ignore_unknown_config_fields",
+            description = "Whether to ignore unknown properties when deserializing the connector configuration.",
+            required = false)
+    public Boolean ignoreUnknownConfigFields = false;
+
+
     private Server server;
     private RuntimeSpawner runtimeSpawner;
     private ThreadRuntimeFactory containerFactory;
@@ -177,6 +183,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL
         instanceConfig.setClusterName(clusterName);
         instanceConfig.setMaxPendingAsyncRequests(maxPendingAsyncRequests);
         instanceConfig.setExposePulsarAdminClientEnabled(exposePulsarAdminClientEnabled);
+        instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields);
         Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder();
         if (functionDetailsJsonString.charAt(0) == '\'') {
             functionDetailsJsonString = functionDetailsJsonString.substring(1);
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
index 53ebfcbfaf068..5392697e9283b 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
@@ -435,10 +435,14 @@ && isNotBlank(authConfig.getClientAuthenticationParameters())) {
         args.add("--metrics_port");
         args.add(String.valueOf(instanceConfig.getMetricsPort()));
 
-        // only the Java instance supports --pending_async_requests right now.
+        // params supported only by the Java instance runtime.
         if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) {
             args.add("--pending_async_requests");
             args.add(String.valueOf(instanceConfig.getMaxPendingAsyncRequests()));
+
+            if (instanceConfig.isIgnoreUnknownConfigFields()) {
+                args.add("--ignore_unknown_config_fields");
+            }
         }
 
         // state storage configs
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
index 3b8ddf774d162..0ed73953d7aa7 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
@@ -738,6 +738,17 @@ public String getFunctionAuthProviderClassName() {
     )
     private List additionalJavaRuntimeArguments = new ArrayList<>();
 
+    @FieldContext(
+            category = CATEGORY_CONNECTORS,
+            doc = "Whether to ignore unknown properties when deserializing the connector configuration. "
+                    + "After upgrading a connector to a new version with a new configuration, "
+                    + "the new configuration may not be compatible with the old connector. "
+                    + "In case of rollback, it's required to also rollback the connector configuration. "
+                    + "Ignoring unknown fields makes possible to keep the new configuration and "
+                    + "only rollback the connector."
+    )
+    private boolean ignoreUnknownConfigFields = false;
+
     public String getFunctionMetadataTopic() {
         return String.format("persistent://%s/%s", pulsarFunctionsNamespace, functionMetadataTopicName);
     }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
index c587a8a734888..03c6eb7921840 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
@@ -221,6 +221,7 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu
         if (workerConfig.getAdditionalJavaRuntimeArguments() != null) {
             instanceConfig.setAdditionalJavaRuntimeArguments(workerConfig.getAdditionalJavaRuntimeArguments());
         }
+        instanceConfig.setIgnoreUnknownConfigFields(workerConfig.isIgnoreUnknownConfigFields());
         return instanceConfig;
     }
 
diff --git a/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml
index 02314241f28e6..2fc04034f655f 100644
--- a/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml
+++ b/pulsar-io/alluxio/src/main/resources/META-INF/services/pulsar-io.yaml
@@ -18,4 +18,5 @@
 #
 name: alluxio
 description: Writes data into Alluxio
-sinkClass: org.apache.pulsar.io.alluxio.sink.AlluxioSink
\ No newline at end of file
+sinkClass: org.apache.pulsar.io.alluxio.sink.AlluxioSink
+sinkConfigClass: org.apache.pulsar.io.alluxio.sink.AlluxioSinkConfig
\ No newline at end of file
diff --git a/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml
index dd6ba976adb10..50cad346b3e2a 100644
--- a/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml
+++ b/pulsar-io/dynamodb/src/main/resources/META-INF/services/pulsar-io.yaml
@@ -20,3 +20,4 @@
 name: dynamodb
 description: DynamoDB connectors
 sourceClass: org.apache.pulsar.io.dynamodb.DynamoDBSource
+sourceConfigClass: org.apache.pulsar.io.dynamodb.DynamoDBSourceConfig
diff --git a/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml
index 1b45638dd4de9..4b80907ee861a 100644
--- a/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml
+++ b/pulsar-io/jdbc/clickhouse/src/main/resources/META-INF/services/pulsar-io.yaml
@@ -20,3 +20,4 @@
 name: jdbc-clickhouse
 description: JDBC sink for ClickHouse
 sinkClass: org.apache.pulsar.io.jdbc.ClickHouseJdbcAutoSchemaSink
+sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig
diff --git a/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml
index 463c8cba1621f..81dd7277e1f15 100644
--- a/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml
+++ b/pulsar-io/jdbc/mariadb/src/main/resources/META-INF/services/pulsar-io.yaml
@@ -20,3 +20,4 @@
 name: jdbc-mariadb
 description: JDBC sink for MariaDB
 sinkClass: org.apache.pulsar.io.jdbc.MariadbJdbcAutoSchemaSink
+sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig
diff --git a/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml
index f6262df6997b4..38c120f089369 100644
--- a/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml
+++ b/pulsar-io/jdbc/openmldb/src/main/resources/META-INF/services/pulsar-io.yaml
@@ -20,3 +20,4 @@
 name: jdbc-openmldb
 description: JDBC sink for OpenMLDB
 sinkClass: org.apache.pulsar.io.jdbc.OpenMLDBJdbcAutoSchemaSink
+sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig
diff --git a/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml
index a8d192a45688f..282ac8a023016 100644
--- a/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml
+++ b/pulsar-io/jdbc/postgres/src/main/resources/META-INF/services/pulsar-io.yaml
@@ -20,3 +20,4 @@
 name: jdbc-postgres
 description: JDBC sink for PostgreSQL
 sinkClass: org.apache.pulsar.io.jdbc.PostgresJdbcAutoSchemaSink
+sinkConfigClass: org.apache.pulsar.io.jdbc.JdbcSinkConfig
\ No newline at end of file

From 77f00413fffe2132c7bc9b1af740b57e221df6d3 Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Fri, 28 Apr 2023 04:53:11 +0300
Subject: [PATCH 237/404] [improve][misc] Change PIP issue template (#19832)

---
 .github/ISSUE_TEMPLATE/pip.md  | 167 +++++++++++++++++++++++++++++++++
 .github/ISSUE_TEMPLATE/pip.yml |  82 ----------------
 2 files changed, 167 insertions(+), 82 deletions(-)
 create mode 100644 .github/ISSUE_TEMPLATE/pip.md
 delete mode 100644 .github/ISSUE_TEMPLATE/pip.yml

diff --git a/.github/ISSUE_TEMPLATE/pip.md b/.github/ISSUE_TEMPLATE/pip.md
new file mode 100644
index 0000000000000..520826f8b1ceb
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/pip.md
@@ -0,0 +1,167 @@
+---
+name: PIP
+about: Submit a Pulsar Improvement Proposal (PIP)
+title: 'PIP-XYZ: '
+labels: PIP
+---
+
+
+
+# Background knowledge
+
+
+
+# Motivation
+
+
+
+# Goals
+
+## In Scope
+
+
+
+## Out of Scope
+
+
+
+
+# High Level Design
+
+
+
+# Detailed Design
+
+## Design & Implementation Details
+
+
+
+## Public-facing Changes
+
+
+
+### Public API 
+
+
+### Binary protocol
+
+### Configuration
+
+### CLI
+
+### Metrics
+
+
+
+
+# Monitoring
+
+
+
+# Security Considerations
+
+
+# Backward & Forward Compatability
+
+## Revert
+
+
+
+## Upgrade
+
+
+
+# Alternatives
+
+
+
+# General Notes
+
+# Links
+
+
+* Mailing List discussion thread:
+* Mailing List voting thread:
diff --git a/.github/ISSUE_TEMPLATE/pip.yml b/.github/ISSUE_TEMPLATE/pip.yml
deleted file mode 100644
index d494d69947252..0000000000000
--- a/.github/ISSUE_TEMPLATE/pip.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: PIP
-title: "PIP-XYZ: "
-description: Submit a Pulsar Improvement Proposal (PIP)
-labels: [ "PIP" ]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thank you very much for submitting a Pulsar Improvement Proposal (PIP)! Here are instructions for creating a PIP using this issue template.
-
-        Please send a note to the dev@pulsar.apache.org mailing list to start the discussion, using subject prefix `[DISCUSS] PIP-XYZ`. To determine the appropriate PIP number XYZ, inspect the [mailing list](https://lists.apache.org/list.html?dev@pulsar.apache.org) for the most recent PIP. Add 1 to that PIP's number to get your PIP's number.
-
-        Based on the discussion and feedback, some changes might be applied by the author(s) to the text of the proposal.
-
-        Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list. Everyone is welcome to vote on the proposal, though it will considered to be binding only the vote of PMC members. It will be required to have a lazy majority of at least 3 binding +1s votes. The vote should stay open for at least 48 hours.
-
-        When the vote is closed, if the outcome is positive, the state of the proposal is updated and the Pull Requests associated with this proposal can start to get merged into the master branch.
-  - type: textarea
-    attributes:
-      label: Motivation
-      description: |
-        Explain why this change is needed, what benefits it would bring to Apache Pulsar and what problem it's trying to solve.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Goal
-      description: |
-        Define the scope of this proposal. Given the motivation stated above, what are the problems that this proposal is addressing and what other items will be considering out of scope, perhaps to be left to a different PIP.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: API Changes
-      description: |
-        Illustrate all the proposed changes to the API or wire protocol, with examples of all the newly added classes/methods, including Javadoc.
-  - type: textarea
-    attributes:
-      label: Implementation
-      description: |
-        This should be a detailed description of all the changes that are expected to be made. It should be detailed enough that any developer that is familiar with Pulsar internals would be able to understand all the parts of the code changes for this proposal.
-
-        This should also serve as documentation for any person that is trying to understand or debug the behavior of a certain feature.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Security Considerations
-      description: |
-        A detailed description of the security details that ought to be considered for the PIP. This is most relevant for any new HTTP endpoints, new Pulsar Protocol Commands, and new security features. The goal is to describe details like which role will have permission to perform an action.
-
-        If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Alternatives
-      description: |
-        If there are alternatives that were already considered by the authors or, after the discussion, by the community, and were rejected, please list them here along with the reason why they were rejected.
-  - type: textarea
-    attributes:
-      label: Anything else?
-  - type: markdown
-    attributes:
-      value: "Thanks for completing our form!"

From 8b7aa6312b2af49d0c6edde85c6b0108007a0a96 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Fri, 28 Apr 2023 16:46:51 +0800
Subject: [PATCH 238/404] [fix][admin] Fix examine messages if total message is
 zero (#20152)

---
 .../pulsar/broker/admin/impl/PersistentTopicsBase.java    | 4 ++++
 .../apache/pulsar/broker/admin/PersistentTopicsTest.java  | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 6b163ae04469b..fcade8270cb12 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -3061,6 +3061,10 @@ protected CompletableFuture internalExamineMessageAsync(String initial
                     try {
                         PersistentTopic persistentTopic = (PersistentTopic) topic;
                         long totalMessage = persistentTopic.getNumberOfEntries();
+                        if (totalMessage <= 0) {
+                            throw new RestException(Status.PRECONDITION_FAILED,
+                                    "Could not examine messages due to the total message is zero");
+                        }
                         PositionImpl startPosition = persistentTopic.getFirstPosition();
 
                         long messageToSkip = initialPositionLocal.equals("earliest") ? messagePositionLocal :
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java
index f80d9863a26a6..284e50c830286 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java
@@ -1151,6 +1151,14 @@ public void testExamineMessage() throws Exception {
                             + "topic partition");
         }
 
+        try {
+            admin.topics().examineMessage(topicName + "-partition-0", "earliest", 1);
+            Assert.fail();
+        } catch (PulsarAdminException e) {
+            Assert.assertEquals(e.getMessage(),
+                    "Could not examine messages due to the total message is zero");
+        }
+
         producer.send("message1");
         Assert.assertEquals(
                 new String(admin.topics().examineMessage(topicName + "-partition-0", "earliest", 1).getData()),

From b8543ad979798d89da9f35a9375de7e16b7e5e25 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sat, 29 Apr 2023 01:52:52 +0800
Subject: [PATCH 239/404] [improve][broker] Support cgroup v2 by using
 `jdk.internal.platform.Metrics` in Pulsar Loadbalancer (#16832)

---
 bin/pulsar                                    |  2 +
 buildtools/pom.xml                            |  1 +
 pom.xml                                       |  1 +
 .../broker/loadbalance/LinuxInfoUtils.java    | 66 ++++++++++++++++---
 .../impl/LinuxBrokerHostUsageImpl.java        |  2 +-
 .../loadbalance/SimpleBrokerStartTest.java    | 26 ++++++++
 .../impl/LinuxBrokerHostUsageImplTest.java    | 37 ++++++++++-
 7 files changed, 122 insertions(+), 13 deletions(-)

diff --git a/bin/pulsar b/bin/pulsar
index a033de947d4b3..e3b22caced52e 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -307,6 +307,8 @@ if [[ -z "$IS_JAVA_8" ]]; then
   OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED"
   # MBeanStatsGenerator
   OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED"
+  # LinuxInfoUtils
+  OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED"
 fi
 
 OPTS="-cp $PULSAR_CLASSPATH $OPTS"
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index 04b25cced42ac..a21618004e31a 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -58,6 +58,7 @@
     
       --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
       --add-opens java.base/java.lang=ALL-UNNAMED 
+      --add-opens java.base/jdk.internal.platform=ALL-UNNAMED 
     
   
 
diff --git a/pom.xml b/pom.xml
index aef380c5cd09c..c5d7092c7b4ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -109,6 +109,7 @@ flexible messaging model and an intuitive client API.
       --add-opens java.base/sun.net=ALL-UNNAMED 
       --add-opens java.management/sun.management=ALL-UNNAMED 
       --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED 
+      --add-opens java.base/jdk.internal.platform=ALL-UNNAMED 
     
     true
     4
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
index 42ef264b6db04..1405979ae3b12 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
@@ -18,7 +18,9 @@
  */
 package org.apache.pulsar.broker.loadbalance;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -45,6 +47,7 @@ public class LinuxInfoUtils {
     private static final String CGROUPS_CPU_USAGE_PATH = "/sys/fs/cgroup/cpu/cpuacct.usage";
     private static final String CGROUPS_CPU_LIMIT_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
     private static final String CGROUPS_CPU_LIMIT_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
+
     // proc states
     private static final String PROC_STAT_PATH = "/proc/stat";
     private static final String NIC_PATH = "/sys/class/net/";
@@ -52,6 +55,30 @@ public class LinuxInfoUtils {
     private static final int ARPHRD_ETHER = 1;
     private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed";
 
+    private static Object /*jdk.internal.platform.Metrics*/ metrics;
+    private static Method getMetricsProviderMethod;
+    private static Method getCpuQuotaMethod;
+    private static Method getCpuPeriodMethod;
+    private static Method getCpuUsageMethod;
+
+    static {
+        try {
+            metrics = Class.forName("jdk.internal.platform.Container").getMethod("metrics")
+                    .invoke(null);
+            if (metrics != null) {
+                getMetricsProviderMethod = metrics.getClass().getMethod("getProvider");
+                getMetricsProviderMethod.setAccessible(true);
+                getCpuQuotaMethod = metrics.getClass().getMethod("getCpuQuota");
+                getCpuQuotaMethod.setAccessible(true);
+                getCpuPeriodMethod = metrics.getClass().getMethod("getCpuPeriod");
+                getCpuPeriodMethod.setAccessible(true);
+                getCpuUsageMethod = metrics.getClass().getMethod("getCpuUsage");
+                getCpuUsageMethod.setAccessible(true);
+            }
+        } catch (Throwable e) {
+            log.warn("Failed to get runtime metrics", e);
+        }
+    }
 
     /**
      * Determine whether the OS is the linux kernel.
@@ -66,9 +93,14 @@ public static boolean isLinux() {
      */
     public static boolean isCGroupEnabled() {
         try {
-            return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH));
+            if (metrics == null) {
+                return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH));
+            }
+            String provider = (String) getMetricsProviderMethod.invoke(metrics);
+            log.info("[LinuxInfo] The system metrics provider is: {}", provider);
+            return provider.contains("cgroup");
         } catch (Exception e) {
-            log.warn("[LinuxInfo] Failed to check cgroup CPU usage file: {}", e.getMessage());
+            log.warn("[LinuxInfo] Failed to check cgroup CPU: {}", e.getMessage());
             return false;
         }
     }
@@ -81,13 +113,21 @@ public static boolean isCGroupEnabled() {
     public static double getTotalCpuLimit(boolean isCGroupsEnabled) {
         if (isCGroupsEnabled) {
             try {
-                long quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH));
-                long period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH));
+                long quota;
+                long period;
+                if (metrics != null && getCpuQuotaMethod != null && getCpuPeriodMethod != null) {
+                    quota = (long) getCpuQuotaMethod.invoke(metrics);
+                    period = (long) getCpuPeriodMethod.invoke(metrics);
+                } else {
+                    quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH));
+                    period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH));
+                }
+
                 if (quota > 0) {
                     return 100.0 * quota / period;
                 }
-            } catch (IOException e) {
-                log.warn("[LinuxInfo] Failed to read CPU quotas from cgroups", e);
+            } catch (Exception e) {
+                log.warn("[LinuxInfo] Failed to read CPU quotas from cgroup", e);
                 // Fallback to availableProcessors
             }
         }
@@ -99,11 +139,14 @@ public static double getTotalCpuLimit(boolean isCGroupsEnabled) {
      * Get CGroup cpu usage.
      * @return Cpu usage
      */
-    public static double getCpuUsageForCGroup() {
+    public static long getCpuUsageForCGroup() {
         try {
+            if (metrics != null && getCpuUsageMethod != null) {
+                return (long) getCpuUsageMethod.invoke(metrics);
+            }
             return readLongFromFile(Paths.get(CGROUPS_CPU_USAGE_PATH));
-        } catch (IOException e) {
-            log.error("[LinuxInfo] Failed to read CPU usage from {}", CGROUPS_CPU_USAGE_PATH, e);
+        } catch (Exception e) {
+            log.error("[LinuxInfo] Failed to read CPU usage from cgroup", e);
             return -1;
         }
     }
@@ -291,6 +334,11 @@ enum Operstate {
         UP
     }
 
+    @VisibleForTesting
+    public static Object getMetrics() {
+        return metrics;
+    }
+
     @AllArgsConstructor
     public enum NICUsageType {
         // transport
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
index 318f37f7f7a97..5d2b7bdd09adf 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
@@ -140,7 +140,7 @@ private double getTotalCpuUsage(double elapsedTimeSeconds) {
     }
 
     private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) {
-        double usage = getCpuUsageForCGroup();
+        double usage = (double) getCpuUsageForCGroup();
         double currentUsage = usage - lastCpuUsage;
         lastCpuUsage = usage;
         return 100 * currentUsage / elapsedTimeSeconds / TimeUnit.SECONDS.toNanos(1);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java
index 6de2eb90f12ef..28dde8b7f559d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java
@@ -20,6 +20,8 @@
 
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Optional;
 import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
@@ -28,6 +30,7 @@
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
 @Slf4j
@@ -96,4 +99,27 @@ public void testNoNICSpeed() throws Exception {
     }
 
 
+    @Test
+    public void testCGroupMetrics() {
+        if (!LinuxInfoUtils.isLinux()) {
+            return;
+        }
+
+        boolean existsCGroup = Files.exists(Paths.get("/sys/fs/cgroup"));
+        boolean cGroupEnabled = LinuxInfoUtils.isCGroupEnabled();
+        Assert.assertEquals(cGroupEnabled, existsCGroup);
+
+        double totalCpuLimit = LinuxInfoUtils.getTotalCpuLimit(cGroupEnabled);
+        log.info("totalCpuLimit: {}", totalCpuLimit);
+        Assert.assertTrue(totalCpuLimit > 0.0);
+
+        if (cGroupEnabled) {
+            Assert.assertNotNull(LinuxInfoUtils.getMetrics());
+
+            long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup();
+            log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup);
+            Assert.assertTrue(cpuUsageForCGroup > 0);
+        }
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java
index 39298dce0b16a..563f707c445b2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java
@@ -18,15 +18,19 @@
  */
 package org.apache.pulsar.broker.loadbalance.impl;
 
-import lombok.Cleanup;
-import org.testng.Assert;
-import org.testng.annotations.Test;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
 
+@Slf4j
 public class LinuxBrokerHostUsageImplTest {
 
     @Test
@@ -42,4 +46,31 @@ public void checkOverrideBrokerNicSpeedGbps() {
         double totalLimit = linuxBrokerHostUsage.getTotalNicLimitWithConfiguration(nics);
         Assert.assertEquals(totalLimit, 3.0 * 1000 * 1000 * 3);
     }
+
+    @Test
+    public void testCpuUsage() throws InterruptedException {
+        if (!LinuxInfoUtils.isLinux()) {
+            return;
+        }
+
+        @Cleanup("shutdown")
+        ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+        LinuxBrokerHostUsageImpl linuxBrokerHostUsage =
+                new LinuxBrokerHostUsageImpl(Integer.MAX_VALUE, Optional.empty(), executorService);
+
+        linuxBrokerHostUsage.calculateBrokerHostUsage();
+        TimeUnit.SECONDS.sleep(1);
+        linuxBrokerHostUsage.calculateBrokerHostUsage();
+
+        double usage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().usage;
+        double limit = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().limit;
+        float percentUsage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().percentUsage();
+
+        Assert.assertTrue(usage > 0);
+        Assert.assertTrue(limit > 0);
+        Assert.assertTrue(limit >= usage);
+        Assert.assertTrue(percentUsage > 0);
+
+        log.info("usage: {}, limit: {}, percentUsage: {}", usage, limit, percentUsage);
+    }
 }

From d135c4a115038dc61f8fe2d230cb1f0c02239f92 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Sat, 29 Apr 2023 01:54:05 +0800
Subject: [PATCH 240/404] [improve] [broker] Skip split boundle if only one
 broker (#20190)

Co-authored-by: Zixuan Liu 
---
 .../impl/ModularLoadManagerImpl.java          |  2 +-
 .../broker/stats/PrometheusMetricsTest.java   |  9 +++
 .../client/api/BrokerServiceLookupTest.java   | 58 +++++++++++++++++++
 3 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index c6f406a7c8a5c..30a2ef5cdf250 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -744,7 +744,7 @@ public boolean shouldAntiAffinityNamespaceUnload(String namespace, String bundle
     public void checkNamespaceBundleSplit() {
 
         if (!conf.isLoadBalancerAutoBundleSplitEnabled() || pulsar.getLeaderElectionService() == null
-                || !pulsar.getLeaderElectionService().isLeader()) {
+                || !pulsar.getLeaderElectionService().isLeader() || knownBrokers.size() <= 1) {
             return;
         }
         final boolean unloadSplitBundles = pulsar.getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index f5ae8459f1803..bf141e10aa1b9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -38,6 +38,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -80,6 +81,7 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.compaction.Compactor;
+import org.apache.zookeeper.CreateMode;
 import org.awaitility.Awaitility;
 import org.mockito.Mockito;
 import org.testng.Assert;
@@ -772,6 +774,10 @@ public void testBundlesMetrics() throws Exception {
             c1.acknowledge(c1.receive());
         }
 
+        // Mock another broker to make split task work.
+        String mockedBroker = "/loadbalance/brokers/127.0.0.1:0";
+        mockZooKeeper.create(mockedBroker, new byte[]{0}, Collections.emptyList(), CreateMode.EPHEMERAL);
+
         pulsar.getBrokerService().updateRates();
         Awaitility.await().untilAsserted(() -> assertTrue(pulsar.getBrokerService().getBundleStats().size() > 0));
         ModularLoadManagerWrapper loadManager = (ModularLoadManagerWrapper)pulsar.getLoadManager().get();
@@ -796,6 +802,9 @@ public void testBundlesMetrics() throws Exception {
         assertTrue(metrics.containsKey("pulsar_lb_bandwidth_out_usage"));
 
         assertTrue(metrics.containsKey("pulsar_lb_bundles_split_total"));
+
+        // cleanup.
+        mockZooKeeper.delete(mockedBroker, 0);
     }
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
index 2af9f450dd3b1..8597e0a87997a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
@@ -67,6 +67,7 @@
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
 import lombok.Cleanup;
+import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
@@ -80,6 +81,7 @@
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
@@ -781,6 +783,62 @@ public void testModularLoadManagerSplitBundle() throws Exception {
         }
     }
 
+    @Test(timeOut = 20000)
+    public void testSkipSplitBundleIfOnlyOneBroker() throws Exception {
+
+        log.info("-- Starting {} test --", methodName);
+        final String loadBalancerName = conf.getLoadManagerClassName();
+        final int defaultNumberOfNamespaceBundles = conf.getDefaultNumberOfNamespaceBundles();
+        final int loadBalancerNamespaceBundleMaxTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
+
+        final String namespace = "my-property/my-ns";
+        final String topicName1 = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_");
+        final String topicName2 = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_");
+        try {
+            // configure broker with ModularLoadManager.
+            stopBroker();
+            conf.setDefaultNumberOfNamespaceBundles(1);
+            conf.setLoadBalancerNamespaceBundleMaxTopics(1);
+            conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+            startBroker();
+            final ModularLoadManagerWrapper modularLoadManagerWrapper =
+                    (ModularLoadManagerWrapper) pulsar.getLoadManager().get();
+            final ModularLoadManagerImpl modularLoadManager =
+                    (ModularLoadManagerImpl) modularLoadManagerWrapper.getLoadManager();
+
+            // Create one topic and trigger tasks, then verify there is only one bundle now.
+            Consumer consumer1 = pulsarClient.newConsumer().topic(topicName1)
+                    .subscriptionName("my-subscriber-name").subscribe();
+            List bounldes1 = pulsar.getNamespaceService().getNamespaceBundleFactory()
+                    .getBundles(NamespaceName.get(namespace)).getBundles();
+            pulsar.getBrokerService().updateRates();
+            pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
+            pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper();
+            modularLoadManager.updateAll();
+            assertEquals(bounldes1.size(), 1);
+
+            // Create the second topic and trigger tasks, then verify the split task will be skipped.
+            Consumer consumer2 = pulsarClient.newConsumer().topic(topicName2)
+                    .subscriptionName("my-subscriber-name").subscribe();
+            pulsar.getBrokerService().updateRates();
+            pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
+            pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper();
+            modularLoadManager.updateAll();
+            List bounldes2 = pulsar.getNamespaceService().getNamespaceBundleFactory()
+                    .getBundles(NamespaceName.get(namespace)).getBundles();
+            assertEquals(bounldes2.size(), 1);
+
+            consumer1.close();
+            consumer2.close();
+            admin.topics().delete(topicName1, false);
+            admin.topics().delete(topicName2, false);
+        } finally {
+            conf.setDefaultNumberOfNamespaceBundles(defaultNumberOfNamespaceBundles);
+            conf.setLoadBalancerNamespaceBundleMaxTopics(loadBalancerNamespaceBundleMaxTopics);
+            conf.setLoadManagerClassName(loadBalancerName);
+        }
+    }
+
     @Test(timeOut = 10000)
     public void testPartitionedMetadataWithDeprecatedVersion() throws Exception {
 

From 7ccd1ba119b1349de4f865b0bc0e1065c6769950 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 28 Apr 2023 20:55:19 +0300
Subject: [PATCH 241/404] [improve][ci] Disable Maven http connection pooling
 on CI also for newer Maven versions (#20198)

---
 .github/workflows/ci-go-functions.yaml           | 2 +-
 .github/workflows/ci-maven-cache-update.yaml     | 2 +-
 .github/workflows/ci-owasp-dependency-check.yaml | 2 +-
 .github/workflows/pulsar-ci-flaky.yaml           | 2 +-
 .github/workflows/pulsar-ci.yaml                 | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index a34cb1a90908e..f96a6d6586e6e 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -32,7 +32,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   preconditions:
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index a5a88e2e0b8a6..0bf1c4148b2c9 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -42,7 +42,7 @@ on:
     - cron: '30 */12 * * *'
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   update-maven-dependencies-cache:
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index 8aec501f8d868..9f1ba2cc996f8 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,7 +24,7 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   run-owasp-dependency-check:
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index ea19bc3307819..fdc25ca1d93a2 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -36,7 +36,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 4d4f2902d8f27..fca0928477925 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -36,7 +36,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR

From a362eaa8bd8caf13795aba5ddbdf4edd8cec7bd0 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Sat, 29 Apr 2023 01:57:43 +0800
Subject: [PATCH 242/404] [improve][broker] Improve knownBrokers update in
 ModularLoadManagerImpl (#20196)

---
 .../loadbalance/impl/ModularLoadManagerImpl.java     | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index 30a2ef5cdf250..73b4f318f3a36 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -195,7 +195,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
     private long unloadBundleCount = 0;
 
     private final Lock lock = new ReentrantLock();
-    private Set knownBrokers = ConcurrentHashMap.newKeySet();
+    private final Set knownBrokers = new HashSet<>();
     private Map bundleBrokerAffinityMap;
 
     /**
@@ -480,13 +480,11 @@ public void updateAll() {
         checkNamespaceBundleSplit();
     }
 
-    private void cleanupDeadBrokersData() {
+    private synchronized void cleanupDeadBrokersData() {
         final Set activeBrokers = getAvailableBrokers();
-        final Set knownBrokersCopy = new HashSet<>(this.knownBrokers);
-        Collection newBrokers = CollectionUtils.subtract(activeBrokers, knownBrokersCopy);
-        this.knownBrokers.addAll(newBrokers);
-        Collection deadBrokers = CollectionUtils.subtract(knownBrokersCopy, activeBrokers);
-        this.knownBrokers.removeAll(deadBrokers);
+        Collection deadBrokers = CollectionUtils.subtract(knownBrokers, activeBrokers);
+        this.knownBrokers.clear();
+        this.knownBrokers.addAll(activeBrokers);
         if (pulsar.getLeaderElectionService() != null
                 && pulsar.getLeaderElectionService().isLeader()) {
             deadBrokers.forEach(this::deleteTimeAverageDataFromMetadataStoreAsync);

From 8c963e3ae3359fee824294c590a9c045f4800228 Mon Sep 17 00:00:00 2001
From: Clay Johnson 
Date: Sat, 29 Apr 2023 20:26:10 -0500
Subject: [PATCH 243/404] [improve][build] Capture local build scans on
 ge.apache.org to benefit from deep build insights (#20187)

---
 .github/actions/gradle-enterprise/action.yml | 1 -
 .gitignore                                   | 1 -
 .mvn/{ge-extensions.xml => extensions.xml}   | 2 +-
 3 files changed, 1 insertion(+), 3 deletions(-)
 rename .mvn/{ge-extensions.xml => extensions.xml} (97%)

diff --git a/.github/actions/gradle-enterprise/action.yml b/.github/actions/gradle-enterprise/action.yml
index 935e76d3cd645..a11e59899c756 100644
--- a/.github/actions/gradle-enterprise/action.yml
+++ b/.github/actions/gradle-enterprise/action.yml
@@ -29,7 +29,6 @@ runs:
     - run: |
         if [[ -n "${{ inputs.token }}" ]]; then
             echo "::group::Configuring Gradle Enterprise for build"
-            cp .mvn/ge-extensions.xml .mvn/extensions.xml
             echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV
             echo "::endgroup::"  
         fi
diff --git a/.gitignore b/.gitignore
index c584baaa0a0b8..cd00c44200059 100644
--- a/.gitignore
+++ b/.gitignore
@@ -97,4 +97,3 @@ test-reports/
 
 # Gradle Enterprise
 .mvn/.gradle-enterprise/
-.mvn/extensions.xml
diff --git a/.mvn/ge-extensions.xml b/.mvn/extensions.xml
similarity index 97%
rename from .mvn/ge-extensions.xml
rename to .mvn/extensions.xml
index 1c7a1611c1bcc..3c03b32a23775 100644
--- a/.mvn/ge-extensions.xml
+++ b/.mvn/extensions.xml
@@ -24,7 +24,7 @@
   
     com.gradle
     gradle-enterprise-maven-extension
-    1.16.3
+    1.17
   
   
     com.gradle

From e8d63952a40b79499eb3fd524f80db6bc986ed34 Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Sun, 30 Apr 2023 12:58:53 +0300
Subject: [PATCH 244/404] [fix][docs] Remove old template inlined (#20208)

---
 wiki/proposals/PIP.md | 35 +----------------------------------
 1 file changed, 1 insertion(+), 34 deletions(-)

diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md
index 159c9199895d1..fc68905726eb8 100644
--- a/wiki/proposals/PIP.md
+++ b/wiki/proposals/PIP.md
@@ -116,38 +116,5 @@ Some examples:
 
 ## Template for a PIP design doc
 
-```
-## Motivation
+Read [the template file](/.github/ISSUE_TEMPLATE/pip.md).
 
-Explain why this change is needed, what benefits it would bring to Apache Pulsar
-and what problem it's trying to solve.
-
-## Goal
-
-Define the scope of this proposal. Given the motivation stated above, what are
-the problems that this proposal is addressing and what other items will be
-considering out of scope, perhaps to be left to a different PIP.
-
-## API Changes
-
-Illustrate all the proposed changes to the API or wire protocol, with examples
-of all the newly added classes/methods, including Javadoc.
-
-## Implementation
-
-This should be a detailed description of all the changes that are
-expected to be made. It should be detailed enough that any developer that is
-familiar with Pulsar internals would be able to understand all the parts of the
-code changes for this proposal.
-
-This should also serve as documentation for any person that is trying to
-understand or debug the behavior of a certain feature.
-
-
-## Reject Alternatives
-
-If there are alternatives that were already considered by the authors or,
-after the discussion, by the community, and were rejected, please list them
-here along with the reason why they were rejected.
-
-```

From a22700df665a9ea0a8db98334b6d4337e5e490f3 Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 1 May 2023 16:47:36 +0800
Subject: [PATCH 245/404] [fix][client] SchemaDefinition handle
 JSR310_CONVERSION_ENABLED property (#20201)

Signed-off-by: tison 
---
 .../schema/SchemaDefinitionBuilderImpl.java   |  2 +-
 .../impl/schema/KeyValueSchemaTest.java       | 34 +++++++++++++++++--
 .../pulsar/functions/source/TopicSchema.java  |  2 --
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java
index c50aac5c39821..93869a3bb9e38 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java
@@ -129,7 +129,7 @@ public SchemaDefinitionBuilder withProperties(Map properties)
         if (properties.containsKey(ALWAYS_ALLOW_NULL)) {
             alwaysAllowNull = Boolean.parseBoolean(properties.get(ALWAYS_ALLOW_NULL));
         }
-        if (properties.containsKey(ALWAYS_ALLOW_NULL)) {
+        if (properties.containsKey(JSR310_CONVERSION_ENABLED)) {
             jsr310ConversionEnabled = Boolean.parseBoolean(properties.get(JSR310_CONVERSION_ENABLED));
         }
         return this;
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java
index b40fa86e8bf95..078c52c5f2e11 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java
@@ -69,7 +69,7 @@ public void testAllowNullAvroSchemaCreate() {
     }
 
     @Test
-    public void testFillParametersToSchemainfo() {
+    public void testFillParametersToSchemaInfo() {
         Map keyProperties = new TreeMap<>();
         keyProperties.put("foo.key1", "value");
         keyProperties.put("foo.key2", "value");
@@ -89,7 +89,6 @@ public void testFillParametersToSchemainfo() {
                         .build());
 
         Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema);
-
         assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.type"), String.valueOf(SchemaType.AVRO));
         assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.properties"),
                 "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\",\"foo.key1\":\"value\",\"foo.key2\":\"value\"}");
@@ -98,6 +97,37 @@ public void testFillParametersToSchemainfo() {
                 "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\",\"bar.key\":\"key\"}");
     }
 
+    @Test
+    public void testOverwriteSchemaDefaultProperties() {
+        Map keyProperties = new TreeMap<>();
+        keyProperties.put("foo.key1", "value");
+        keyProperties.put("foo.key2", "value");
+        keyProperties.put(SchemaDefinitionBuilderImpl.ALWAYS_ALLOW_NULL, "false");
+        keyProperties.put(SchemaDefinitionBuilderImpl.JSR310_CONVERSION_ENABLED, "true");
+
+        Map valueProperties = new TreeMap<>();
+        valueProperties.put("bar.key", "key");
+
+        AvroSchema fooSchema = AvroSchema.of(
+                SchemaDefinition.builder()
+                        .withPojo(Foo.class)
+                        .withProperties(keyProperties)
+                        .build());
+        AvroSchema barSchema = AvroSchema.of(
+                SchemaDefinition.builder()
+                        .withPojo(Bar.class)
+                        .withProperties(valueProperties)
+                        .build());
+
+        Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema);
+        assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.type"), String.valueOf(SchemaType.AVRO));
+        assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("key.schema.properties"),
+                "{\"__alwaysAllowNull\":\"false\",\"__jsr310ConversionEnabled\":\"true\",\"foo.key1\":\"value\",\"foo.key2\":\"value\"}");
+        assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("value.schema.type"), String.valueOf(SchemaType.AVRO));
+        assertEquals(keyValueSchema1.getSchemaInfo().getProperties().get("value.schema.properties"),
+                "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\",\"bar.key\":\"key\"}");
+    }
+
     @Test
     public void testNotAllowNullAvroSchemaCreate() {
         AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).withAlwaysAllowNull(false).build());
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
index 57f49fed0cac9..8ed177cba3b19 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
@@ -44,8 +44,6 @@
 @Slf4j
 public class TopicSchema {
 
-    public static final String JSR_310_CONVERSION_ENABLED = "jsr310ConversionEnabled";
-    public static final String ALWAYS_ALLOW_NULL = "alwaysAllowNull";
     private final Map> cachedSchemas = new HashMap<>();
     private final PulsarClient client;
 

From 010bd508cecae501fa812be6df4b3f7d33ec4be6 Mon Sep 17 00:00:00 2001
From: Clay Johnson 
Date: Thu, 4 May 2023 06:06:13 -0500
Subject: [PATCH 246/404] [improve][ci] Replace handmade action to configure
 Gradle Enterprise (#20218)

---
 .github/actions/gradle-enterprise/action.yml  | 35 ----------
 .github/workflows/ci-maven-cache-update.yaml  |  6 +-
 .../workflows/ci-owasp-dependency-check.yaml  |  7 +-
 .github/workflows/pulsar-ci-flaky.yaml        |  6 +-
 .github/workflows/pulsar-ci.yaml              | 68 ++++---------------
 5 files changed, 16 insertions(+), 106 deletions(-)
 delete mode 100644 .github/actions/gradle-enterprise/action.yml

diff --git a/.github/actions/gradle-enterprise/action.yml b/.github/actions/gradle-enterprise/action.yml
deleted file mode 100644
index a11e59899c756..0000000000000
--- a/.github/actions/gradle-enterprise/action.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: Configure Gradle Enterprise integration
-description: Configure Gradle Enterprise when secret GE_ACCESS_TOKEN is available
-inputs:
-  token:
-    description: 'The token for accessing Gradle Enterprise'
-    required: true
-runs:
-  using: composite
-  steps:
-    - run: |
-        if [[ -n "${{ inputs.token }}" ]]; then
-            echo "::group::Configuring Gradle Enterprise for build"
-            echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV
-            echo "::endgroup::"  
-        fi
-      shell: bash
\ No newline at end of file
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index 0bf1c4148b2c9..15fefaf3f1645 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -49,6 +49,7 @@ jobs:
     name: Update Maven dependency cache for ${{ matrix.name }}
     env:
       JOB_NAME: Update Maven dependency cache for ${{ matrix.name }}
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 45
 
@@ -77,11 +78,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Detect changed files
         if: ${{ github.event_name != 'schedule' }}
         id:   changes
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index 9f1ba2cc996f8..06edbae51adde 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -32,6 +32,7 @@ jobs:
     name: Check ${{ matrix.branch }}
     env:
       JOB_NAME: Check ${{ matrix.branch }}
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-20.04
     timeout-minutes: 45
     strategy:
@@ -57,12 +58,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        if: ${{ matrix.branch == 'master' }}
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Cache local Maven repository
         uses: actions/cache@v3
         timeout-minutes: 5
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index fdc25ca1d93a2..555ebdb17292f 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -94,6 +94,7 @@ jobs:
     env:
       JOB_NAME: Flaky tests suite
       COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-20.04
     timeout-minutes: 100
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -104,11 +105,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index fca0928477925..b92599581cd83 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -95,6 +95,7 @@ jobs:
     name: Build and License check
     env:
       JOB_NAME: Build and License check
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -105,11 +106,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -175,6 +171,7 @@ jobs:
     env:
       JOB_NAME: CI - Unit - ${{ matrix.name }}
       COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-20.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'build-and-license-check']
@@ -211,11 +208,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -391,6 +383,8 @@ jobs:
     timeout-minutes: 60
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -398,11 +392,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -469,6 +458,7 @@ jobs:
     env:
       JOB_NAME: CI - Integration - ${{ matrix.name }}
       PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     strategy:
       fail-fast: false
       matrix:
@@ -513,11 +503,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -721,11 +706,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
 
@@ -739,6 +719,8 @@ jobs:
     timeout-minutes: 60
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -746,11 +728,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -851,6 +828,7 @@ jobs:
     env:
       JOB_NAME: CI - System - ${{ matrix.name }}
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     strategy:
       fail-fast: false
       matrix:
@@ -886,11 +864,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -1081,6 +1054,7 @@ jobs:
     env:
       JOB_NAME: CI Flaky - System - ${{ matrix.name }}
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     strategy:
       fail-fast: false
       matrix:
@@ -1098,11 +1072,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -1208,11 +1177,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
 
@@ -1226,6 +1190,8 @@ jobs:
     timeout-minutes: 120
     needs: ['preconditions', 'integration-tests']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -1233,11 +1199,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Cache Maven dependencies
         uses: actions/cache@v3
         timeout-minutes: 5
@@ -1264,6 +1225,8 @@ jobs:
     timeout-minutes: 120
     needs: [ 'preconditions', 'integration-tests' ]
     if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -1271,11 +1234,6 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Configure Gradle Enterprise
-        uses: ./.github/actions/gradle-enterprise
-        with:
-          token: ${{ secrets.GE_ACCESS_TOKEN }}
-
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}

From 2b515ffb389c4b4fe3cb5a9c5f3d7eb0a4c9ef99 Mon Sep 17 00:00:00 2001
From: ran 
Date: Fri, 5 May 2023 08:48:59 +0800
Subject: [PATCH 247/404] [fix][broker] Fix the reason label of authentication
 metrics (#20030)

---
 .../AuthenticationProviderAthenz.java         | 20 ++++++++++--
 .../oidc/AuthenticationProviderOpenID.java    |  2 +-
 .../AuthenticationProvider.java               |  5 +++
 .../AuthenticationProviderBasic.java          | 21 +++++++++++--
 .../AuthenticationProviderList.java           | 20 +++++++-----
 .../AuthenticationProviderTls.java            | 12 +++++--
 .../AuthenticationProviderToken.java          | 31 ++++++++++++-------
 .../metrics/AuthenticationMetrics.java        | 16 ++++++++++
 .../broker/stats/PrometheusMetricsTest.java   |  6 ++--
 9 files changed, 103 insertions(+), 30 deletions(-)

diff --git a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java
index 2e062b87a8325..652a922b9a5ad 100644
--- a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java
+++ b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java
@@ -43,6 +43,15 @@ public class AuthenticationProviderAthenz implements AuthenticationProvider {
     private List domainNameList = null;
     private int allowedOffset = 30;
 
+    public enum ErrorCode {
+        UNKNOWN,
+        NO_CLIENT,
+        NO_TOKEN,
+        NO_PUBLIC_KEY,
+        DOMAIN_MISMATCH,
+        INVALID_TOKEN,
+    }
+
     @Override
     public void initialize(ServiceConfiguration config) throws IOException {
         String domainNames;
@@ -81,11 +90,13 @@ public String getAuthMethodName() {
     public String authenticate(AuthenticationDataSource authData) throws AuthenticationException {
         SocketAddress clientAddress;
         String roleToken;
+        ErrorCode errorCode = ErrorCode.UNKNOWN;
         try {
 
             if (authData.hasDataFromPeer()) {
                 clientAddress = authData.getPeerAddress();
             } else {
+                errorCode = ErrorCode.NO_CLIENT;
                 throw new AuthenticationException("Authentication data source does not have a client address");
             }
 
@@ -94,13 +105,16 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
             } else if (authData.hasDataFromHttp()) {
                 roleToken = authData.getHttpHeader(AuthZpeClient.ZPE_TOKEN_HDR);
             } else {
+                errorCode = ErrorCode.NO_TOKEN;
                 throw new AuthenticationException("Authentication data source does not have a role token");
             }
 
             if (roleToken == null) {
+                errorCode = ErrorCode.NO_TOKEN;
                 throw new AuthenticationException("Athenz token is null, can't authenticate");
             }
             if (roleToken.isEmpty()) {
+                errorCode = ErrorCode.NO_TOKEN;
                 throw new AuthenticationException("Athenz RoleToken is empty, Server is Using Athenz Authentication");
             }
             if (log.isDebugEnabled()) {
@@ -110,6 +124,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
             RoleToken token = new RoleToken(roleToken);
 
             if (!domainNameList.contains(token.getDomain())) {
+                errorCode = ErrorCode.DOMAIN_MISMATCH;
                 throw new AuthenticationException(
                         String.format("Athenz RoleToken Domain mismatch, Expected: %s, Found: %s",
                                 domainNameList.toString(), token.getDomain()));
@@ -120,6 +135,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
                 PublicKey ztsPublicKey = AuthZpeClient.getZtsPublicKey(token.getKeyId());
 
                 if (ztsPublicKey == null) {
+                    errorCode = ErrorCode.NO_PUBLIC_KEY;
                     throw new AuthenticationException("Unable to retrieve ZTS Public Key");
                 }
 
@@ -128,13 +144,13 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
                     AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName());
                     return token.getPrincipal();
                 } else {
+                    errorCode = ErrorCode.INVALID_TOKEN;
                     throw new AuthenticationException(
                             String.format("Athenz Role Token Not Authenticated from Client: %s", clientAddress));
                 }
             }
         } catch (AuthenticationException exception) {
-            AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(),
-                    exception.getMessage());
+            incrementFailureMetric(errorCode);
             throw exception;
         }
     }
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
index ae4774b6f6b6b..00ec09bd1817f 100644
--- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
@@ -447,7 +447,7 @@ DecodedJWT verifyJWT(PublicKey publicKey,
     }
 
     static void incrementFailureMetric(AuthenticationExceptionCode code) {
-        AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, "token", code.toString());
+        AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, AUTH_METHOD_NAME, code);
     }
 
     /**
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java
index 109259537a494..7862a35b5e871 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java
@@ -30,6 +30,7 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics;
 import org.apache.pulsar.common.api.AuthData;
 import org.apache.pulsar.common.util.FutureUtil;
 
@@ -143,6 +144,10 @@ default CompletableFuture authenticateHttpRequestAsync(HttpServletReque
         }
     }
 
+    default void incrementFailureMetric(Enum errorCode) {
+        AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), errorCode);
+    }
+
     /**
      * Set response, according to passed in request.
      * and return whether we should do following chain.doFilter or not.
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java
index 3c4759fec7da9..ca5150c9bdb60 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java
@@ -46,6 +46,14 @@ public class AuthenticationProviderBasic implements AuthenticationProvider {
     private static final String CONF_PULSAR_PROPERTY_KEY = "basicAuthConf";
     private Map users;
 
+    private enum ErrorCode {
+        UNKNOWN,
+        EMPTY_AUTH_DATA,
+        INVALID_HEADER,
+        INVALID_AUTH_DATA,
+        INVALID_TOKEN,
+    }
+
     @Override
     public void close() throws IOException {
         // noop
@@ -104,9 +112,10 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
         String userId = authParams.getUserId();
         String password = authParams.getPassword();
         String msg = "Unknown user or invalid password";
-
+        ErrorCode errorCode = ErrorCode.UNKNOWN;
         try {
             if (users.get(userId) == null) {
+                errorCode = ErrorCode.INVALID_AUTH_DATA;
                 throw new AuthenticationException(msg);
             }
 
@@ -117,15 +126,16 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
                 List splitEncryptedPassword = Arrays.asList(encryptedPassword.split("\\$"));
                 if (splitEncryptedPassword.size() != 4 || !encryptedPassword
                         .equals(Md5Crypt.apr1Crypt(password.getBytes(), splitEncryptedPassword.get(2)))) {
+                    errorCode = ErrorCode.INVALID_TOKEN;
                     throw new AuthenticationException(msg);
                 }
                 // For crypt algorithm
             } else if (!encryptedPassword.equals(Crypt.crypt(password.getBytes(), encryptedPassword.substring(0, 2)))) {
+                errorCode = ErrorCode.INVALID_TOKEN;
                 throw new AuthenticationException(msg);
             }
         } catch (AuthenticationException exception) {
-            AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(),
-                    exception.getMessage());
+            incrementFailureMetric(errorCode);
             throw exception;
         }
         AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName());
@@ -144,24 +154,29 @@ public AuthParams(AuthenticationDataSource authData) throws AuthenticationExcept
                 String rawAuthToken = authData.getHttpHeader(HTTP_HEADER_NAME);
                 // parsing and validation
                 if (StringUtils.isBlank(rawAuthToken) || !rawAuthToken.toUpperCase().startsWith("BASIC ")) {
+                    incrementFailureMetric(ErrorCode.INVALID_HEADER);
                     throw new AuthenticationException("Authentication token has to be started with \"Basic \"");
                 }
                 String[] splitRawAuthToken = rawAuthToken.split(" ");
                 if (splitRawAuthToken.length != 2) {
+                    incrementFailureMetric(ErrorCode.INVALID_HEADER);
                     throw new AuthenticationException("Base64 encoded token is not found");
                 }
 
                 try {
                     authParams = new String(Base64.getDecoder().decode(splitRawAuthToken[1]));
                 } catch (Exception e) {
+                    incrementFailureMetric(ErrorCode.INVALID_HEADER);
                     throw new AuthenticationException("Base64 decoding is failure: " + e.getMessage());
                 }
             } else {
+                incrementFailureMetric(ErrorCode.EMPTY_AUTH_DATA);
                 throw new AuthenticationException("Authentication data source does not have data");
             }
 
             String[] parsedAuthParams = authParams.split(":");
             if (parsedAuthParams.length != 2) {
+                incrementFailureMetric(ErrorCode.INVALID_AUTH_DATA);
                 throw new AuthenticationException("Base64 decoded params are invalid");
             }
 
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java
index 16d6f9859a0b4..663a6253f4460 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java
@@ -44,9 +44,15 @@ private interface AuthProcessor {
 
     }
 
+    private enum ErrorCode {
+        UNKNOWN,
+        AUTH_REQUIRED,
+    }
+
     static  T applyAuthProcessor(List processors, AuthProcessor authFunc)
         throws AuthenticationException {
         AuthenticationException authenticationException = null;
+        String errorCode = ErrorCode.UNKNOWN.name();
         for (W ap : processors) {
             try {
                 return authFunc.apply(ap);
@@ -56,19 +62,19 @@ static  T applyAuthProcessor(List processors, AuthProcessor authF
                 }
                 // Store the exception so we can throw it later instead of a generic one
                 authenticationException = ae;
+                errorCode = ap.getClass().getSimpleName() + "-INVALID-AUTH";
             }
         }
 
         if (null == authenticationException) {
             AuthenticationMetrics.authenticateFailure(
                     AuthenticationProviderList.class.getSimpleName(),
-                    "authentication-provider-list", "Authentication required");
+                    "authentication-provider-list", ErrorCode.AUTH_REQUIRED);
             throw new AuthenticationException("Authentication required");
         } else {
-            AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(),
-                    "authentication-provider-list",
-                    authenticationException.getMessage() != null
-                            ? authenticationException.getMessage() : "Authentication required");
+            AuthenticationMetrics.authenticateFailure(
+                    AuthenticationProviderList.class.getSimpleName(),
+                    "authentication-provider-list", errorCode);
             throw authenticationException;
         }
 
@@ -129,7 +135,7 @@ private void authenticateRemainingAuthStates(CompletableFuture authCha
                     previousException = new AuthenticationException("Authentication required");
                 }
                 AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(),
-                        "authentication-provider-list", "Authentication required");
+                        "authentication-provider-list", ErrorCode.AUTH_REQUIRED);
                 authChallengeFuture.completeExceptionally(previousException);
                 return;
             }
@@ -235,7 +241,7 @@ private void authenticateRemainingAuthProviders(CompletableFuture roleFu
                 previousException = new AuthenticationException("Authentication required");
             }
             AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(),
-                    "authentication-provider-list", "Authentication required");
+                    "authentication-provider-list", ErrorCode.AUTH_REQUIRED);
             roleFuture.completeExceptionally(previousException);
             return;
         }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java
index 47e5316bce9ce..a4c44121b4b96 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java
@@ -27,6 +27,12 @@
 
 public class AuthenticationProviderTls implements AuthenticationProvider {
 
+    private enum ErrorCode {
+        UNKNOWN,
+        INVALID_CERTS,
+        INVALID_CN, // invalid common name
+    }
+
     @Override
     public void close() throws IOException {
         // noop
@@ -45,6 +51,7 @@ public String getAuthMethodName() {
     @Override
     public String authenticate(AuthenticationDataSource authData) throws AuthenticationException {
         String commonName = null;
+        ErrorCode errorCode = ErrorCode.UNKNOWN;
         try {
             if (authData.hasDataFromTls()) {
                 /**
@@ -72,6 +79,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
                 // CN=Steve Kille,O=Isode Limited,C=GB
                 Certificate[] certs = authData.getTlsCertificates();
                 if (null == certs) {
+                    errorCode = ErrorCode.INVALID_CERTS;
                     throw new AuthenticationException("Failed to get TLS certificates from client");
                 }
                 String distinguishedName = ((X509Certificate) certs[0]).getSubjectX500Principal().getName();
@@ -85,12 +93,12 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat
             }
 
             if (commonName == null) {
+                errorCode = ErrorCode.INVALID_CN;
                 throw new AuthenticationException("Client unable to authenticate with TLS certificate");
             }
             AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName());
         } catch (AuthenticationException exception) {
-            AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(),
-                    exception.getMessage());
+            incrementFailureMetric(errorCode);
             throw exception;
         }
         return commonName;
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java
index 67e1f39a38924..f8992b21ff49f 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java
@@ -106,6 +106,12 @@ public class AuthenticationProviderToken implements AuthenticationProvider {
     private String confTokenAudienceSettingName;
     private String confTokenAllowedClockSkewSecondsSettingName;
 
+    public enum ErrorCode {
+        INVALID_AUTH_DATA,
+        INVALID_TOKEN,
+        INVALID_AUDIENCES,
+    }
+
     @Override
     public void close() throws IOException {
         // noop
@@ -158,19 +164,18 @@ public String getAuthMethodName() {
 
     @Override
     public String authenticate(AuthenticationDataSource authData) throws AuthenticationException {
+        String token;
         try {
             // Get Token
-            String token;
             token = getToken(authData);
-            // Parse Token by validating
-            String role = getPrincipal(authenticateToken(token));
-            AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName());
-            return role;
         } catch (AuthenticationException exception) {
-            AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(),
-                    exception.getMessage());
+            incrementFailureMetric(ErrorCode.INVALID_AUTH_DATA);
             throw exception;
         }
+        // Parse Token by validating
+        String role = getPrincipal(authenticateToken(token));
+        AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName());
+        return role;
     }
 
     @Override
@@ -241,16 +246,19 @@ private Jwt authenticateToken(final String token) throws Authenticati
                     List audiences = (List) object;
                     // audience not contains this broker, throw exception.
                     if (audiences.stream().noneMatch(audienceInToken -> audienceInToken.equals(audience))) {
-                        throw new AuthenticationException("Audiences in token: [" + String.join(", ", audiences)
-                                                          + "] not contains this broker: " + audience);
+                        incrementFailureMetric(ErrorCode.INVALID_AUDIENCES);
+                        throw new AuthenticationException("Audiences in token: ["
+                                + String.join(", ", audiences) + "] not contains this broker: " + audience);
                     }
                 } else if (object instanceof String) {
                     if (!object.equals(audience)) {
-                        throw new AuthenticationException("Audiences in token: [" + object
-                                                          + "] not contains this broker: " + audience);
+                        incrementFailureMetric(ErrorCode.INVALID_AUDIENCES);
+                        throw new AuthenticationException(
+                                "Audiences in token: [" + object + "] not contains this broker: " + audience);
                     }
                 } else {
                     // should not reach here.
+                    incrementFailureMetric(ErrorCode.INVALID_AUDIENCES);
                     throw new AuthenticationException("Audiences in token is not in expected format: " + object);
                 }
             }
@@ -264,6 +272,7 @@ private Jwt authenticateToken(final String token) throws Authenticati
             if (e instanceof ExpiredJwtException) {
                 expiredTokenMetrics.inc();
             }
+            incrementFailureMetric(ErrorCode.INVALID_TOKEN);
             throw new AuthenticationException("Failed to authentication token: " + e.getMessage());
         }
     }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java
index 0a095235f3cb3..5faaccbe15716 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java
@@ -43,11 +43,27 @@ public static void authenticateSuccess(String providerName, String authMethod) {
 
     /**
      * Log authenticate failure event to the authentication metrics.
+     *
+     * This method is deprecated due to the label "reason" is a potential infinite value.
+     * @deprecated See {@link #authenticateFailure(String, String, Enum)} ()}
+     *
      * @param providerName The short class name of the provider
      * @param authMethod Authentication method name.
      * @param reason Failure reason.
      */
+    @Deprecated
     public static void authenticateFailure(String providerName, String authMethod, String reason) {
         authFailuresMetrics.labels(providerName, authMethod, reason).inc();
     }
+
+    /**
+     * Log authenticate failure event to the authentication metrics.
+     * @param providerName The short class name of the provider
+     * @param authMethod Authentication method name.
+     * @param errorCode Error code.
+     */
+    public static void authenticateFailure(String providerName, String authMethod, Enum errorCode) {
+        authFailuresMetrics.labels(providerName, authMethod, errorCode.name()).inc();
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index bf141e10aa1b9..7a23e56fe0b41 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -1379,15 +1379,12 @@ public void testAuthMetrics() throws IOException, AuthenticationException {
         conf.setProperties(properties);
         provider.initialize(conf);
 
-        String authExceptionMessage = "";
-
         try {
             provider.authenticate(new AuthenticationDataSource() {
             });
             fail("Should have failed");
         } catch (AuthenticationException e) {
             // expected, no credential passed
-            authExceptionMessage = e.getMessage();
         }
 
         String token = AuthTokenUtils.createToken(secretKey, "subject", Optional.empty());
@@ -1424,7 +1421,8 @@ public String getCommandData() {
         boolean haveFailed = false;
         for (Metric metric : cm) {
             if (Objects.equals(metric.tags.get("auth_method"), "token")
-                    && Objects.equals(metric.tags.get("reason"), authExceptionMessage)
+                    && Objects.equals(metric.tags.get("reason"),
+                    AuthenticationProviderToken.ErrorCode.INVALID_AUTH_DATA.name())
                     && Objects.equals(metric.tags.get("provider_name"), provider.getClass().getSimpleName())) {
                 haveFailed = true;
             }

From 0dd238abf3798ba1f5b83182c1c4fb36e6ba6511 Mon Sep 17 00:00:00 2001
From: Zike Yang 
Date: Fri, 5 May 2023 15:31:03 +0800
Subject: [PATCH 248/404] [cleanup][build] Bumped version to 3.1.0-SNAPSHOT
 (#20216)

Signed-off-by: Zike Yang 
---
 bouncy-castle/bc/pom.xml                              | 2 +-
 bouncy-castle/bcfips-include-test/pom.xml             | 2 +-
 bouncy-castle/bcfips/pom.xml                          | 2 +-
 bouncy-castle/pom.xml                                 | 2 +-
 buildtools/pom.xml                                    | 4 ++--
 distribution/io/pom.xml                               | 2 +-
 distribution/offloaders/pom.xml                       | 2 +-
 distribution/pom.xml                                  | 2 +-
 distribution/server/pom.xml                           | 2 +-
 distribution/shell/pom.xml                            | 2 +-
 docker/pom.xml                                        | 2 +-
 docker/pulsar-all/pom.xml                             | 2 +-
 docker/pulsar/pom.xml                                 | 2 +-
 jclouds-shaded/pom.xml                                | 2 +-
 managed-ledger/pom.xml                                | 2 +-
 pom.xml                                               | 4 ++--
 pulsar-broker-auth-athenz/pom.xml                     | 2 +-
 pulsar-broker-auth-oidc/pom.xml                       | 2 +-
 pulsar-broker-auth-sasl/pom.xml                       | 2 +-
 pulsar-broker-common/pom.xml                          | 2 +-
 pulsar-broker/pom.xml                                 | 2 +-
 pulsar-client-1x-base/pom.xml                         | 2 +-
 pulsar-client-1x-base/pulsar-client-1x/pom.xml        | 2 +-
 pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +-
 pulsar-client-admin-api/pom.xml                       | 2 +-
 pulsar-client-admin-shaded/pom.xml                    | 2 +-
 pulsar-client-admin/pom.xml                           | 2 +-
 pulsar-client-all/pom.xml                             | 2 +-
 pulsar-client-api/pom.xml                             | 2 +-
 pulsar-client-auth-athenz/pom.xml                     | 2 +-
 pulsar-client-auth-sasl/pom.xml                       | 2 +-
 pulsar-client-messagecrypto-bc/pom.xml                | 2 +-
 pulsar-client-shaded/pom.xml                          | 2 +-
 pulsar-client-tools-api/pom.xml                       | 2 +-
 pulsar-client-tools-customcommand-example/pom.xml     | 2 +-
 pulsar-client-tools-test/pom.xml                      | 2 +-
 pulsar-client-tools/pom.xml                           | 2 +-
 pulsar-client/pom.xml                                 | 2 +-
 pulsar-common/pom.xml                                 | 2 +-
 pulsar-config-validation/pom.xml                      | 2 +-
 pulsar-functions/api-java/pom.xml                     | 2 +-
 pulsar-functions/instance/pom.xml                     | 2 +-
 pulsar-functions/java-examples-builtin/pom.xml        | 2 +-
 pulsar-functions/java-examples/pom.xml                | 2 +-
 pulsar-functions/localrun-shaded/pom.xml              | 2 +-
 pulsar-functions/localrun/pom.xml                     | 2 +-
 pulsar-functions/pom.xml                              | 2 +-
 pulsar-functions/proto/pom.xml                        | 2 +-
 pulsar-functions/runtime-all/pom.xml                  | 2 +-
 pulsar-functions/runtime/pom.xml                      | 2 +-
 pulsar-functions/secrets/pom.xml                      | 2 +-
 pulsar-functions/utils/pom.xml                        | 2 +-
 pulsar-functions/worker/pom.xml                       | 2 +-
 pulsar-io/aerospike/pom.xml                           | 2 +-
 pulsar-io/alluxio/pom.xml                             | 2 +-
 pulsar-io/aws/pom.xml                                 | 2 +-
 pulsar-io/batch-data-generator/pom.xml                | 2 +-
 pulsar-io/batch-discovery-triggerers/pom.xml          | 2 +-
 pulsar-io/canal/pom.xml                               | 2 +-
 pulsar-io/cassandra/pom.xml                           | 2 +-
 pulsar-io/common/pom.xml                              | 2 +-
 pulsar-io/core/pom.xml                                | 2 +-
 pulsar-io/data-generator/pom.xml                      | 2 +-
 pulsar-io/debezium/core/pom.xml                       | 2 +-
 pulsar-io/debezium/mongodb/pom.xml                    | 2 +-
 pulsar-io/debezium/mssql/pom.xml                      | 2 +-
 pulsar-io/debezium/mysql/pom.xml                      | 2 +-
 pulsar-io/debezium/oracle/pom.xml                     | 2 +-
 pulsar-io/debezium/pom.xml                            | 2 +-
 pulsar-io/debezium/postgres/pom.xml                   | 2 +-
 pulsar-io/docs/pom.xml                                | 2 +-
 pulsar-io/dynamodb/pom.xml                            | 2 +-
 pulsar-io/elastic-search/pom.xml                      | 2 +-
 pulsar-io/file/pom.xml                                | 2 +-
 pulsar-io/flume/pom.xml                               | 2 +-
 pulsar-io/hbase/pom.xml                               | 2 +-
 pulsar-io/hdfs2/pom.xml                               | 2 +-
 pulsar-io/hdfs3/pom.xml                               | 2 +-
 pulsar-io/http/pom.xml                                | 2 +-
 pulsar-io/influxdb/pom.xml                            | 2 +-
 pulsar-io/jdbc/clickhouse/pom.xml                     | 2 +-
 pulsar-io/jdbc/core/pom.xml                           | 2 +-
 pulsar-io/jdbc/mariadb/pom.xml                        | 2 +-
 pulsar-io/jdbc/openmldb/pom.xml                       | 2 +-
 pulsar-io/jdbc/pom.xml                                | 2 +-
 pulsar-io/jdbc/postgres/pom.xml                       | 2 +-
 pulsar-io/jdbc/sqlite/pom.xml                         | 2 +-
 pulsar-io/kafka-connect-adaptor-nar/pom.xml           | 2 +-
 pulsar-io/kafka-connect-adaptor/pom.xml               | 2 +-
 pulsar-io/kafka/pom.xml                               | 2 +-
 pulsar-io/kinesis/pom.xml                             | 2 +-
 pulsar-io/mongo/pom.xml                               | 2 +-
 pulsar-io/netty/pom.xml                               | 2 +-
 pulsar-io/nsq/pom.xml                                 | 2 +-
 pulsar-io/pom.xml                                     | 2 +-
 pulsar-io/rabbitmq/pom.xml                            | 2 +-
 pulsar-io/redis/pom.xml                               | 2 +-
 pulsar-io/solr/pom.xml                                | 2 +-
 pulsar-io/twitter/pom.xml                             | 2 +-
 pulsar-metadata/pom.xml                               | 2 +-
 pulsar-package-management/bookkeeper-storage/pom.xml  | 2 +-
 pulsar-package-management/core/pom.xml                | 2 +-
 pulsar-package-management/filesystem-storage/pom.xml  | 2 +-
 pulsar-package-management/pom.xml                     | 2 +-
 pulsar-proxy/pom.xml                                  | 2 +-
 pulsar-sql/pom.xml                                    | 2 +-
 pulsar-sql/presto-distribution/pom.xml                | 2 +-
 pulsar-sql/presto-pulsar-plugin/pom.xml               | 2 +-
 pulsar-sql/presto-pulsar/pom.xml                      | 2 +-
 pulsar-testclient/pom.xml                             | 2 +-
 pulsar-transaction/common/pom.xml                     | 2 +-
 pulsar-transaction/coordinator/pom.xml                | 2 +-
 pulsar-transaction/pom.xml                            | 2 +-
 pulsar-websocket/pom.xml                              | 2 +-
 structured-event-log/pom.xml                          | 2 +-
 testmocks/pom.xml                                     | 2 +-
 tests/bc_2_0_0/pom.xml                                | 2 +-
 tests/bc_2_0_1/pom.xml                                | 2 +-
 tests/bc_2_6_0/pom.xml                                | 2 +-
 tests/docker-images/java-test-functions/pom.xml       | 2 +-
 tests/docker-images/java-test-image/pom.xml           | 2 +-
 tests/docker-images/java-test-plugins/pom.xml         | 2 +-
 tests/docker-images/latest-version-image/pom.xml      | 2 +-
 tests/docker-images/pom.xml                           | 2 +-
 tests/integration/pom.xml                             | 2 +-
 tests/pom.xml                                         | 2 +-
 tests/pulsar-client-admin-shade-test/pom.xml          | 2 +-
 tests/pulsar-client-all-shade-test/pom.xml            | 2 +-
 tests/pulsar-client-shade-test/pom.xml                | 2 +-
 tiered-storage/file-system/pom.xml                    | 2 +-
 tiered-storage/jcloud/pom.xml                         | 2 +-
 tiered-storage/pom.xml                                | 2 +-
 132 files changed, 134 insertions(+), 134 deletions(-)

diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml
index cc7e7952b69f0..d5882b4659528 100644
--- a/bouncy-castle/bc/pom.xml
+++ b/bouncy-castle/bc/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     bouncy-castle-parent
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml
index 44f0ada4630d7..3b8c6754c3fa9 100644
--- a/bouncy-castle/bcfips-include-test/pom.xml
+++ b/bouncy-castle/bcfips-include-test/pom.xml
@@ -24,7 +24,7 @@
   
     org.apache.pulsar
     bouncy-castle-parent
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml
index bd5d64bd84191..a07e5e19907f2 100644
--- a/bouncy-castle/bcfips/pom.xml
+++ b/bouncy-castle/bcfips/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     bouncy-castle-parent
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml
index 9f8ace79b77c3..daefeb83b5371 100644
--- a/bouncy-castle/pom.xml
+++ b/bouncy-castle/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     pulsar
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index a21618004e31a..46099df4d39ff 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -31,12 +31,12 @@
 
   org.apache.pulsar
   buildtools
-  3.0.0-SNAPSHOT
+  3.1.0-SNAPSHOT
   jar
   Pulsar Build Tools
 
   
-    ${maven.build.timestamp}
+    2023-05-03T02:53:27Z
     1.8
     1.8
     3.0.0-M3
diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
index 99105bef950d5..568d76922bf4e 100644
--- a/distribution/io/pom.xml
+++ b/distribution/io/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     distribution
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml
index 1e86758ed5a6a..d23ebec2ef26d 100644
--- a/distribution/offloaders/pom.xml
+++ b/distribution/offloaders/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     distribution
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 9782f269284bf..36a3fa1c5835a 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     pulsar
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index 6b225bfc00c0b..f804c9c54b9cd 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     distribution
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml
index b38baee4257ba..9e3134a75e5bf 100644
--- a/distribution/shell/pom.xml
+++ b/distribution/shell/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     distribution
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/docker/pom.xml b/docker/pom.xml
index 4d3b05fe33a76..882240925ef24 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -26,7 +26,7 @@
   
     org.apache.pulsar
     pulsar
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
   
   docker-images
   Apache Pulsar :: Docker Images
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index 2edd6c776dcf5..7a2f492632135 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -23,7 +23,7 @@
   
     org.apache.pulsar
     docker-images
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
   
   4.0.0
   pulsar-all-docker-image
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 8c3b868315535..e1c1503a3f381 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -23,7 +23,7 @@
   
     org.apache.pulsar
     docker-images
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
   
   4.0.0
   pulsar-docker-image
diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml
index d4138ea041317..dfb155c2d5a7d 100644
--- a/jclouds-shaded/pom.xml
+++ b/jclouds-shaded/pom.xml
@@ -26,7 +26,7 @@
   
     org.apache.pulsar
     pulsar
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
index 2a7b9c576d318..a8cb560b7b376 100644
--- a/managed-ledger/pom.xml
+++ b/managed-ledger/pom.xml
@@ -25,7 +25,7 @@
   
     org.apache.pulsar
     pulsar
-    3.0.0-SNAPSHOT
+    3.1.0-SNAPSHOT
     ..
   
 
diff --git a/pom.xml b/pom.xml
index c5d7092c7b4ec..5e19823ffea77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
   org.apache.pulsar
   pulsar
 
-  3.0.0-SNAPSHOT
+  3.1.0-SNAPSHOT
 
   Pulsar
   Pulsar is a distributed pub-sub messaging platform with a very
@@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API.
 
     UTF-8
     UTF-8
-    ${maven.build.timestamp}
+    2023-05-03T02:53:27Z
     true
 
     
     3.2.13
     1.1.1
-    7.7.0
+    7.7.1
     3.12.4
     3.25.0-GA
     1.5.0
@@ -273,14 +273,13 @@ flexible messaging model and an intuitive client API.
     3.0.0
     4.1
     1.0
-    3.1.0
+    3.3.0
     
     
-    
-    3.0.0-M3
-    3.4.2
-    3.10.1
-    3.4.0
+    3.1.0
+    3.5.0
+    3.11.0
+    3.5.0
     2.3.0
     3.4.1
     3.1.0
@@ -288,7 +287,7 @@ flexible messaging model and an intuitive client API.
     1.3.4
     3.1.2
     4.0.2
-    3.4.3
+    3.5.3
     1.7.0
     0.8.8
     4.7.3.0
@@ -1489,7 +1488,6 @@ flexible messaging model and an intuitive client API.
           UTF-8
           true
           true
-          true
           
           false
           
@@ -1542,6 +1540,13 @@ flexible messaging model and an intuitive client API.
             
           
         
+        
+          
+            org.apache.maven.surefire
+            surefire-testng
+            ${surefire.version}
+          
+        
       
 
       
diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml
index 3aba6f55d99e1..a3a3de19202c2 100644
--- a/pulsar-client-tools-customcommand-example/pom.xml
+++ b/pulsar-client-tools-customcommand-example/pom.xml
@@ -64,13 +64,6 @@
           true
         
       
-      
-        org.sonatype.plugins
-        nexus-staging-maven-plugin
-        
-          true
-        
-      
     
   
 

From ea56197d18d8d1f5b06c26246d920cb736d4e6a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Wed, 10 May 2023 13:26:16 +0200
Subject: [PATCH 263/404] [fix][monitor] topic with double quote breaks the
 prometheus format (#20230)

---
 .../prometheus/PrometheusMetricStreams.java   |  6 ++++-
 .../broker/stats/PrometheusMetricsTest.java   | 26 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java
index df107b6d7d9c4..93cbad4e19503 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java
@@ -41,7 +41,11 @@ void writeSample(String metricName, Number value, String... labelsAndValuesArray
         SimpleTextOutputStream stream = initGaugeType(metricName);
         stream.write(metricName).write('{');
         for (int i = 0; i < labelsAndValuesArray.length; i += 2) {
-            stream.write(labelsAndValuesArray[i]).write("=\"").write(labelsAndValuesArray[i + 1]).write('\"');
+            String labelValue = labelsAndValuesArray[i + 1];
+            if (labelValue != null) {
+                labelValue = labelValue.replace("\"", "\\\"");
+            }
+            stream.write(labelsAndValuesArray[i]).write("=\"").write(labelValue).write('\"');
             if (i + 2 != labelsAndValuesArray.length) {
                 stream.write(',');
             }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index 7a23e56fe0b41..a7a28afd8ac64 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -1953,4 +1953,30 @@ public String toString() {
         }
     }
 
+    @Test
+    public void testEscapeLabelValue() throws Exception {
+        String ns1 = "prop/ns-abc1";
+        admin.namespaces().createNamespace(ns1);
+        String topic = "persistent://" + ns1 + "/\"mytopic";
+        admin.topics().createNonPartitionedTopic(topic);
+
+        @Cleanup
+        final Consumer consumer = pulsarClient.newConsumer()
+                .subscriptionName("sub")
+                .topic(topic)
+                .subscribe();
+        @Cleanup
+        ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, true, false,
+                false, statsOut);
+        String metricsStr = statsOut.toString();
+        final List subCountLines = metricsStr.lines()
+                .filter(line -> line.startsWith("pulsar_subscriptions_count"))
+                .collect(Collectors.toList());
+        System.out.println(subCountLines);
+        assertEquals(subCountLines.size(), 1);
+        assertEquals(subCountLines.get(0),
+                "pulsar_subscriptions_count{cluster=\"test\",namespace=\"prop/ns-abc1\",topic=\"persistent://prop/ns-abc1/\\\"mytopic\"} 1");
+    }
+
 }

From c32caba54a5ab2c01c092ab952343f4ac405da35 Mon Sep 17 00:00:00 2001
From: Raghavender Mittapalli <14803749+syk-coder@users.noreply.github.com>
Date: Wed, 10 May 2023 18:05:17 +0530
Subject: [PATCH 264/404] [fix][broker] Fix default bundle size used while
 setting bookie affinity (#20250)

---
 .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java   | 2 +-
 .../pulsar/broker/service/BrokerBookieIsolationTest.java      | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 19f8d7c437ded..8c72d0b0286d3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -883,7 +883,7 @@ protected void internalSetBookieAffinityGroup(BookieAffinityGroupData bookieAffi
                         policies -> new LocalPolicies(policies.bundles,
                                 bookieAffinityGroup,
                                 policies.namespaceAntiAffinityGroup))
-                        .orElseGet(() -> new LocalPolicies(defaultBundle(),
+                        .orElseGet(() -> new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()),
                                 bookieAffinityGroup,
                                 null));
                 log.info("[{}] Successfully updated local-policies configuration: namespace={}, map={}", clientAppId(),
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java
index 36d4053ae497f..951892f4ebfbc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java
@@ -156,6 +156,7 @@ public void testBookieIsolation() throws Exception {
         config.setBrokerServicePort(Optional.of(0));
         config.setAdvertisedAddress("localhost");
         config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups);
+        config.setDefaultNumberOfNamespaceBundles(8);
 
         config.setManagedLedgerDefaultEnsembleSize(2);
         config.setManagedLedgerDefaultWriteQuorum(2);
@@ -207,6 +208,9 @@ public void testBookieIsolation() throws Exception {
                         .bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups)
                         .build());
 
+        //Checks the namespace bundles after setting the bookie affinity
+        assertEquals(admin.namespaces().getBundles(ns2).getNumBundles(), config.getDefaultNumberOfNamespaceBundles());
+
         try {
             admin.namespaces().getBookieAffinityGroup(ns1);
             fail("ns1 should have no bookie affinity group set");

From f04252b84a8b594ea371645e367c491aa8e2dd80 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Wed, 10 May 2023 21:15:37 +0800
Subject: [PATCH 265/404] [fix][fn] Support multiple input topics for Go
 runtime (#20000)

---
 pulsar-function-go/conf/conf.go               |  18 +-
 pulsar-function-go/conf/conf.yaml             |   6 +-
 pulsar-function-go/pf/instance.go             |   4 +-
 pulsar-function-go/pf/instanceConf.go         |  31 ++--
 .../instance/go/GoInstanceConfig.java         |   5 +
 .../src/main/resources/findbugsExclude.xml    |  10 ++
 .../functions/runtime/RuntimeUtils.java       |  13 +-
 .../functions/PulsarFunctionsTest.java        | 160 +++++++++++-------
 .../functions/PulsarFunctionsTestBase.java    |   2 +
 .../functions/go/PulsarFunctionsGoTest.java   |   5 +
 .../java/PulsarFunctionsJavaTest.java         |   4 +-
 .../python/PulsarFunctionsPythonTest.java     |   8 +-
 12 files changed, 179 insertions(+), 87 deletions(-)

diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go
index c2ff443fcc40c..d52b886b540f9 100644
--- a/pulsar-function-go/conf/conf.go
+++ b/pulsar-function-go/conf/conf.go
@@ -50,8 +50,8 @@ type Conf struct {
 	SecretsMap           string `json:"secretsMap" yaml:"secretsMap"`
 	Runtime              int32  `json:"runtime" yaml:"runtime"`
 	//Deprecated
-	AutoACK              bool   `json:"autoAck" yaml:"autoAck"`
-	Parallelism          int32  `json:"parallelism" yaml:"parallelism"`
+	AutoACK     bool  `json:"autoAck" yaml:"autoAck"`
+	Parallelism int32 `json:"parallelism" yaml:"parallelism"`
 	//source config
 	SubscriptionType     int32  `json:"subscriptionType" yaml:"subscriptionType"`
 	TimeoutMs            uint64 `json:"timeoutMs" yaml:"timeoutMs"`
@@ -59,10 +59,16 @@ type Conf struct {
 	CleanupSubscription  bool   `json:"cleanupSubscription"  yaml:"cleanupSubscription"`
 	SubscriptionPosition int32  `json:"subscriptionPosition" yaml:"subscriptionPosition"`
 	//source input specs
-	SourceSpecTopic            string `json:"sourceSpecsTopic" yaml:"sourceSpecsTopic"`
-	SourceSchemaType           string `json:"sourceSchemaType" yaml:"sourceSchemaType"`
-	IsRegexPatternSubscription bool   `json:"isRegexPatternSubscription" yaml:"isRegexPatternSubscription"`
-	ReceiverQueueSize          int32  `json:"receiverQueueSize" yaml:"receiverQueueSize"`
+	SourceInputSpecs map[string]string `json:"sourceInputSpecs" yaml:"sourceInputSpecs"`
+	// for backward compatibility
+	// Deprecated
+	SourceSpecTopic string `json:"sourceSpecsTopic" yaml:"sourceSpecsTopic"`
+	// Deprecated
+	SourceSchemaType string `json:"sourceSchemaType" yaml:"sourceSchemaType"`
+	// Deprecated
+	IsRegexPatternSubscription bool `json:"isRegexPatternSubscription" yaml:"isRegexPatternSubscription"`
+	// Deprecated
+	ReceiverQueueSize int32 `json:"receiverQueueSize" yaml:"receiverQueueSize"`
 	//sink spec config
 	SinkSpecTopic  string `json:"sinkSpecsTopic" yaml:"sinkSpecsTopic"`
 	SinkSchemaType string `json:"sinkSchemaType" yaml:"sinkSchemaType"`
diff --git a/pulsar-function-go/conf/conf.yaml b/pulsar-function-go/conf/conf.yaml
index 59ac9bbd51308..098e33cda1f57 100644
--- a/pulsar-function-go/conf/conf.yaml
+++ b/pulsar-function-go/conf/conf.yaml
@@ -43,10 +43,8 @@ subscriptionName: ""
 cleanupSubscription: false
 subscriptionPosition: 1
 # source input specs
-sourceSpecsTopic: persistent://public/default/topic-01
-sourceSchemaType: ""
-isRegexPatternSubscription: false
-receiverQueueSize: 10
+sourceInputSpecs:
+  persistent://public/default/topic-01: "{\"schemaType\": \"\", \"isRegexPattern\": false, \"receiverQueueSize\": {\"value\": 10}}"
 # sink specs config
 sinkSpecsTopic: persistent://public/default/topic-02
 sinkSchemaType: ""
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index a82273031ecfb..138489444d160 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -439,7 +439,6 @@ func (gi *goInstance) addLogTopicHandler() {
 	}()
 
 	if gi.context.logAppender == nil {
-		log.Error("the logAppender is nil, if you want to use it, please specify `--log-topic` at startup.")
 		return
 	}
 
@@ -571,6 +570,9 @@ func (gi *goInstance) getMatchingMetricFunc() func(lbl *prometheus_client.LabelP
 
 func (gi *goInstance) getMatchingMetricFromRegistry(metricName string) prometheus_client.Metric {
 	filteredMetricFamilies := gi.getFilteredMetricFamilies(metricName)
+	if len(filteredMetricFamilies) == 0 {
+		return prometheus_client.Metric{}
+	}
 	metricFunc := gi.getMatchingMetricFunc()
 	matchingMetric := getFirstMatch(filteredMetricFamilies[0].Metric, metricFunc)
 	return *matchingMetric
diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go
index d60beef29e8d3..9d4cabfae5a9b 100644
--- a/pulsar-function-go/pf/instanceConf.go
+++ b/pulsar-function-go/pf/instanceConf.go
@@ -20,6 +20,7 @@
 package pf
 
 import (
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -44,6 +45,24 @@ type instanceConf struct {
 }
 
 func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf {
+	inputSpecs := make(map[string]*pb.ConsumerSpec)
+	// for backward compatibility
+	if cfg.SourceSpecTopic != "" {
+		inputSpecs[cfg.SourceSpecTopic] = &pb.ConsumerSpec{
+			SchemaType:     cfg.SourceSchemaType,
+			IsRegexPattern: cfg.IsRegexPatternSubscription,
+			ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{
+				Value: cfg.ReceiverQueueSize,
+			},
+		}
+	}
+	for topic, value := range cfg.SourceInputSpecs {
+		spec := &pb.ConsumerSpec{}
+		if err := json.Unmarshal([]byte(value), spec); err != nil {
+			panic(fmt.Sprintf("Failed to unmarshal consume specs: %v", err))
+		}
+		inputSpecs[topic] = spec
+	}
 	instanceConf := &instanceConf{
 		instanceID:                  cfg.InstanceID,
 		funcID:                      cfg.FuncID,
@@ -66,16 +85,8 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf {
 			AutoAck:              cfg.AutoACK,
 			Parallelism:          cfg.Parallelism,
 			Source: &pb.SourceSpec{
-				SubscriptionType: pb.SubscriptionType(cfg.SubscriptionType),
-				InputSpecs: map[string]*pb.ConsumerSpec{
-					cfg.SourceSpecTopic: {
-						SchemaType:     cfg.SourceSchemaType,
-						IsRegexPattern: cfg.IsRegexPatternSubscription,
-						ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{
-							Value: cfg.ReceiverQueueSize,
-						},
-					},
-				},
+				SubscriptionType:     pb.SubscriptionType(cfg.SubscriptionType),
+				InputSpecs:           inputSpecs,
 				TimeoutMs:            cfg.TimeoutMs,
 				SubscriptionName:     cfg.SubscriptionName,
 				CleanupSubscription:  cfg.CleanupSubscription,
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
index abf17bdb1656d..67fe2a41d553d 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.functions.instance.go;
 
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.pulsar.functions.proto.Function;
@@ -53,6 +54,10 @@ public class GoInstanceConfig {
     private boolean cleanupSubscription;
     private int subscriptionPosition = Function.SubscriptionPosition.LATEST.getNumber();
 
+    // value is the json string of ConsumerSpec
+    private Map sourceInputSpecs;
+
+    // for backward compatibility
     private String sourceSpecsTopic = "";
     private String sourceSchemaType = "";
     private boolean isRegexPatternSubscription;
diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml
index 027affbfeb2ec..7fe247d2ab20a 100644
--- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml
@@ -542,4 +542,14 @@
     
     
   
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
 
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
index 5392697e9283b..1d2eacbd77b12 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
@@ -38,6 +38,7 @@
 import java.net.InetAddress;
 import java.net.URL;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -204,9 +205,15 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig,
                 instanceConfig.getFunctionDetails().getSource().getSubscriptionPosition().getNumber());
 
         if (instanceConfig.getFunctionDetails().getSource().getInputSpecsMap() != null) {
-            for (String inputTopic : instanceConfig.getFunctionDetails().getSource().getInputSpecsMap().keySet()) {
-                goInstanceConfig.setSourceSpecsTopic(inputTopic);
+            Map sourceInputSpecs = new HashMap<>();
+            for (Map.Entry entry :
+                    instanceConfig.getFunctionDetails().getSource().getInputSpecsMap().entrySet()) {
+                String topic = entry.getKey();
+                Function.ConsumerSpec spec = entry.getValue();
+                sourceInputSpecs.put(topic, JsonFormat.printer().omittingInsignificantWhitespace().print(spec));
+                goInstanceConfig.setSourceSpecsTopic(topic);
             }
+            goInstanceConfig.setSourceInputSpecs(sourceInputSpecs);
         }
 
         if (instanceConfig.getFunctionDetails().getSource().getTimeoutMs() != 0) {
@@ -304,7 +311,7 @@ public static List getCmd(InstanceConfig instanceConfig,
             }
 
             if (StringUtils.isNotEmpty(functionInstanceClassPath)) {
-               args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, functionInstanceClassPath));
+                args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, functionInstanceClassPath));
             } else {
                 // add complete classpath for broker/worker so that the function instance can load
                 // the functions instance dependencies separately from user code dependencies
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
index eaf66974b982a..6088628aac52b 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
@@ -29,10 +29,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -126,13 +129,14 @@ protected Map produceMessagesToInputTopic(String inputTopicName,
         return kvs;
     }
 
-    protected void testFunctionLocalRun(Runtime runtime) throws  Exception {
+    protected void testFunctionLocalRun(Runtime runtime) throws Exception {
         if (functionRuntimeType == FunctionRuntimeType.THREAD) {
             return;
         }
 
 
-        String inputTopicName = "persistent://public/default/test-function-local-run-" + runtime + "-input-" + randomName(8);
+        String inputTopicName =
+                "persistent://public/default/test-function-local-run-" + runtime + "-input-" + randomName(8);
         String outputTopicName = "test-function-local-run-" + runtime + "-output-" + randomName(8);
 
         final int numMessages = 10;
@@ -377,7 +381,8 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception {
 
         if (runtime == Runtime.PYTHON) {
             submitFunction(
-                    runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema);
+                    runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE,
+                    EXCEPTION_PYTHON_CLASS, schema);
         } else {
             submitFunction(
                     runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema);
@@ -550,7 +555,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception {
         final int numMessages = 10;
 
         // submit the exclamation function
-        switch (runtime){
+        switch (runtime) {
             case JAVA:
                 submitFunction(
                         runtime,
@@ -622,7 +627,8 @@ protected void testPublishFunction(Runtime runtime) throws Exception {
                     .create();
 
             for (int i = 0; i < numMessages; i++) {
-                producer.newMessage().key(String.valueOf(i)).property("count", String.valueOf(i)).value(("message-" + i).getBytes(UTF_8)).send();
+                producer.newMessage().key(String.valueOf(i)).property("count", String.valueOf(i))
+                        .value(("message-" + i).getBytes(UTF_8)).send();
             }
 
             Set expectedMessages = new HashSet<>();
@@ -662,9 +668,10 @@ protected void testPublishFunction(Runtime runtime) throws Exception {
     protected void testExclamationFunction(Runtime runtime,
                                            boolean isTopicPattern,
                                            boolean pyZip,
+                                           boolean multipleInput,
                                            boolean withExtraDeps) throws Exception {
-        if (functionRuntimeType == FunctionRuntimeType.THREAD && runtime == Runtime.PYTHON) {
-            // python can only run on process mode
+        if (functionRuntimeType == FunctionRuntimeType.THREAD && (runtime == Runtime.PYTHON || runtime == Runtime.GO)) {
+            // python&go can only run on process mode
             return;
         }
 
@@ -683,22 +690,9 @@ protected void testExclamationFunction(Runtime runtime,
             admin.topics().createNonPartitionedTopic(outputTopicName);
         }
         if (isTopicPattern) {
-            @Cleanup PulsarClient client = PulsarClient.builder()
-                    .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
-                    .build();
-
-            @Cleanup Consumer consumer1 = client.newConsumer(schema)
-                    .topic(inputTopicName + "1")
-                    .subscriptionType(SubscriptionType.Exclusive)
-                    .subscriptionName("test-sub")
-                    .subscribe();
-
-            @Cleanup Consumer consumer2 = client.newConsumer(schema)
-                    .topic(inputTopicName + "2")
-                    .subscriptionType(SubscriptionType.Exclusive)
-                    .subscriptionName("test-sub")
-                    .subscribe();
             inputTopicName = inputTopicName + ".*";
+        } else if (multipleInput) {
+            inputTopicName = inputTopicName + "1," + inputTopicName + "2";
         }
         String functionName = "test-exclamation-fn-" + randomName(8);
         final int numMessages = 10;
@@ -725,8 +719,11 @@ protected void testExclamationFunction(Runtime runtime,
         // get function status
         getFunctionStatus(functionName, numMessages, true);
 
-        // get function stats
-        getFunctionStats(functionName, numMessages);
+        if (Runtime.GO != runtime) {
+            // TODO: Go runtime doesn't collect `process_latency_ms_1min` metric
+            // get function stats
+            getFunctionStats(functionName, numMessages);
+        }
 
         // update parallelism
         updateFunctionParallelism(functionName, 2);
@@ -787,6 +784,12 @@ private  void submitFunction(Runtime runtime,
             } else {
                 file = EXCLAMATION_PYTHON_FILE;
             }
+        } else if (Runtime.GO == runtime) {
+            if (isPublishFunction) {
+                file = PUBLISH_FUNCTION_GO_FILE;
+            } else {
+                file = EXCLAMATION_GO_FILE;
+            }
         }
 
         submitFunction(runtime, inputTopicName, outputTopicName, functionName, file, functionClass, inputTopicSchema);
@@ -819,7 +822,7 @@ private  void submitFunction(Runtime runtime,
 
         if (StringUtils.isNotEmpty(inputTopicName)) {
             ensureSubscriptionCreated(
-                inputTopicName, String.format("public/default/%s", functionName), inputTopicSchema);
+                    inputTopicName, String.format("public/default/%s", functionName), inputTopicSchema);
         }
 
         CommandGenerator generator;
@@ -853,7 +856,7 @@ private  void submitFunction(Runtime runtime,
         }
         String command = "";
 
-        switch (runtime){
+        switch (runtime) {
             case JAVA:
                 command = generator.generateCreateFunctionCommand();
                 break;
@@ -945,8 +948,17 @@ private  void ensureSubscriptionCreated(String inputTopicName,
         try (PulsarClient client = PulsarClient.builder()
                 .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
                 .build()) {
+            List topics = new ArrayList<>();
+            if (inputTopicName.endsWith(".*")) {
+                topics.add(inputTopicName.substring(0, inputTopicName.length() - 2) + "1");
+                topics.add(inputTopicName.substring(0, inputTopicName.length() - 2) + "2");
+            } else if (inputTopicName.contains(",")) {
+                topics.addAll(Arrays.asList(inputTopicName.split(",")));
+            } else {
+                topics.add(inputTopicName);
+            }
             try (Consumer ignored = client.newConsumer(inputTopicSchema)
-                    .topic(inputTopicName)
+                    .topic(topics.toArray(new String[0]))
                     .subscriptionType(SubscriptionType.Shared)
                     .subscriptionName(subscriptionName)
                     .subscribe()) {
@@ -1042,7 +1054,8 @@ private void getFunctionStats(String functionName, int numMessages) throws Excep
         assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0);
         assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0);
         assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), numMessages);
-        assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), numMessages);
+        assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(),
+                numMessages);
         assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0);
         assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0);
         assertTrue(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency() > 0);
@@ -1071,19 +1084,29 @@ private void getFunctionInfoNotFound(String functionName) throws Exception {
     }
 
     private void checkSubscriptionsCleanup(String topic) throws Exception {
-        try {
-            ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd(
-                    PulsarCluster.ADMIN_SCRIPT,
-                    "topics",
-                    "stats",
-                    topic);
-            TopicStats topicStats = ObjectMapperFactory.getMapper().reader()
-                    .readValue(result.getStdout(), TopicStats.class);
-            assertEquals(topicStats.getSubscriptions().size(), 0);
-
-        } catch (ContainerExecException e) {
-            fail("Command should have exited with non-zero");
+        List topics = new ArrayList<>();
+        if (topic.endsWith(".*")) {
+            topics.add(topic.substring(0, topic.length() - 2) + "1");
+            topics.add(topic.substring(0, topic.length() - 2) + "2");
+        } else if (topic.contains(",")) {
+            topics.addAll(Arrays.asList(topic.split(",")));
+        } else {
+            topics.add(topic);
         }
+        topics.stream().forEach(t -> {
+            try {
+                ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd(
+                        PulsarCluster.ADMIN_SCRIPT,
+                        "topics",
+                        "stats",
+                        t);
+                TopicStats topicStats = ObjectMapperFactory.getMapper().reader()
+                        .readValue(result.getStdout(), TopicStats.class);
+                assertEquals(topicStats.getSubscriptions().size(), 0);
+            } catch (Exception e) {
+                fail("Command should have exited with non-zero");
+            }
+        });
     }
 
     private void checkPublisherCleanup(String topic) throws Exception {
@@ -1145,7 +1168,8 @@ private void doGetFunctionStatus(String functionName, int numMessages, boolean c
             lastInvocationTimeGreaterThanZero = lastInvocationTimeGreaterThanZero
                     || functionStatus.getInstances().get(i).getStatus().getLastInvocationTime() > 0;
             totalMessagesProcessed += functionStatus.getInstances().get(i).getStatus().getNumReceived();
-            totalMessagesSuccessfullyProcessed += functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed();
+            totalMessagesSuccessfullyProcessed +=
+                    functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed();
             if (checkRestarts) {
                 assertEquals(functionStatus.getInstances().get(i).getStatus().getNumRestarts(), 0);
             }
@@ -1238,6 +1262,23 @@ private void publishAndConsumeMessagesBytes(String inputTopic,
                 producer1.send(("message-" + i).getBytes(UTF_8));
             }
 
+            for (int i = numMessages / 2; i < numMessages; i++) {
+                producer2.send(("message-" + i).getBytes(UTF_8));
+            }
+        } else if (inputTopic.contains(",")) {
+            String[] topics = inputTopic.split(",");
+            @Cleanup Producer producer1 = client.newProducer(Schema.BYTES)
+                    .topic(topics[0])
+                    .create();
+
+            @Cleanup Producer producer2 = client.newProducer(Schema.BYTES)
+                    .topic(topics[1])
+                    .create();
+
+            for (int i = 0; i < numMessages / 2; i++) {
+                producer1.send(("message-" + i).getBytes(UTF_8));
+            }
+
             for (int i = numMessages / 2; i < numMessages; i++) {
                 producer2.send(("message-" + i).getBytes(UTF_8));
             }
@@ -1420,7 +1461,8 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception {
                     AVRO_SCHEMA_FUNCTION_PYTHON_FILE,
                     AVRO_SCHEMA_PYTHON_CLASS,
                     Schema.AVRO(AvroTestObject.class),
-                    null, objectMapper.writeValueAsString(inputSpecs), "avro", null, "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject");
+                    null, objectMapper.writeValueAsString(inputSpecs), "avro", null,
+                    "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject");
         }
         log.info("pulsar submitFunction");
 
@@ -1430,7 +1472,7 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception {
         Set expectedSet = new HashSet<>();
 
         log.info("test-avro-schema producer connected: " + producer.isConnected());
-        for (int i = 0 ; i < numMessages ; i++) {
+        for (int i = 0; i < numMessages; i++) {
             AvroTestObject inputObject = new AvroTestObject();
             inputObject.setBaseValue(i);
             MessageId messageId = producer.send(inputObject);
@@ -1457,7 +1499,7 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception {
         });
 
         log.info("test-avro-schema consumer connected: " + consumer.isConnected());
-        for (int i = 0 ; i < numMessages ; i++) {
+        for (int i = 0; i < numMessages; i++) {
             log.info("test-avro-schema consumer receive [{}] start", i);
             Message message = consumer.receive();
             log.info("test-avro-schema consumer receive [{}] over", i);
@@ -1495,7 +1537,8 @@ protected void testInitFunction(Runtime runtime) throws Exception {
         final int numMessages = 10;
 
         // submit the exclamation function
-        submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, InitializableFunction.class.getName(), schema,
+        submitFunction(runtime, inputTopicName, outputTopicName, functionName, null,
+                InitializableFunction.class.getName(), schema,
                 Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null);
 
         // publish and consume result
@@ -1650,7 +1693,8 @@ private void publishAndConsumeMessages(String inputTopic,
     }
 
 
-    protected void testGenericObjectFunction(String function, boolean removeAgeField, boolean keyValue) throws Exception {
+    protected void testGenericObjectFunction(String function, boolean removeAgeField, boolean keyValue)
+            throws Exception {
         log.info("start {} function test ...", function);
 
         String ns = "public/ns-genericobject-" + randomName(8);
@@ -1721,13 +1765,14 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField
                     GenericRecord genericRecord = message.getValue();
                     if (keyValue) {
                         @SuppressWarnings("unchecked")
-                        KeyValue keyValueObject = (KeyValue) genericRecord.getNativeObject();
+                        KeyValue keyValueObject =
+                                (KeyValue) genericRecord.getNativeObject();
                         GenericRecord key = keyValueObject.getKey();
                         GenericRecord value = keyValueObject.getValue();
-                        key.getFields().forEach(f-> {
+                        key.getFields().forEach(f -> {
                             log.info("key field {} value {}", f.getName(), key.getField(f.getName()));
                         });
-                        value.getFields().forEach(f-> {
+                        value.getFields().forEach(f -> {
                             log.info("value field {} value {}", f.getName(), value.getField(f.getName()));
                         });
                         assertEquals(i, key.getField("age"));
@@ -1743,7 +1788,7 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField
                     } else {
                         GenericRecord value = genericRecord;
                         log.info("received value {}", value);
-                        value.getFields().forEach(f-> {
+                        value.getFields().forEach(f -> {
                             log.info("value field {} value {}", f.getName(), value.getField(f.getName()));
                         });
 
@@ -1939,13 +1984,14 @@ private void prepareDataForMergeFunction(String ns,
     }
 
     private  void generateDataByDifferentSchema(String ns,
-                                               String baseTopic,
-                                               PulsarClient pulsarClient,
-                                               Schema schema,
-                                               T data,
-                                               int messageCnt,
-                                               ObjectNode inputSpecNode,
-                                               Map topicMsgCntMap) throws PulsarClientException {
+                                                   String baseTopic,
+                                                   PulsarClient pulsarClient,
+                                                   Schema schema,
+                                                   T data,
+                                                   int messageCnt,
+                                                   ObjectNode inputSpecNode,
+                                                   Map topicMsgCntMap)
+            throws PulsarClientException {
         String topic = ns + "/" + baseTopic;
         Producer producer = pulsarClient.newProducer(schema)
                 .topic(topic)
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
index 033e590d6dd4e..62aa36da1b6be 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
@@ -151,6 +151,8 @@ protected static String getExclamationClass(Runtime runtime,
             } else {
                 return EXCLAMATION_PYTHON_CLASS;
             }
+        } else if (Runtime.GO == runtime) {
+            return null;
         } else {
             throw new IllegalArgumentException("Unsupported runtime : " + runtime);
         }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java
index 9ef2398c55c00..ad8dc475aac20 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java
@@ -34,4 +34,9 @@ public void testGoFunctionLocalRun() throws Exception {
         testFunctionLocalRun(Runtime.GO);
     }
 
+    @Test(groups = {"go_function", "function"})
+    public void testGoExclamationMultiInputsFunction() throws Exception {
+        testExclamationFunction(Runtime.GO, false, false, true, false);
+    }
+
 }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java
index e6f3c67ebcdb7..097a452937d27 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java
@@ -98,12 +98,12 @@ private void testCustomSerdeFunction() throws Exception {
 
     @Test(groups = {"java_function", "function"})
     public void testJavaExclamationFunction() throws Exception {
-        testExclamationFunction(Runtime.JAVA, false, false, false);
+        testExclamationFunction(Runtime.JAVA, false, false, false, false);
     }
 
     @Test(groups = {"java_function", "function"})
     public void testJavaExclamationTopicPatternFunction() throws Exception {
-        testExclamationFunction(Runtime.JAVA, true, false, false);
+        testExclamationFunction(Runtime.JAVA, true, false, false, false);
     }
 
     @Test(groups = {"java_function", "function"})
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java
index 1c75d70498609..87a52d27e8927 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java
@@ -46,22 +46,22 @@ public void testPythonPublishFunction() throws Exception {
 
     @Test(groups = {"python_function", "function"})
     public void testPythonExclamationFunction() throws Exception {
-        testExclamationFunction(Runtime.PYTHON, false, false, false);
+        testExclamationFunction(Runtime.PYTHON, false, false, false, false);
     }
 
     @Test(groups = {"python_function", "function"})
     public void testPythonExclamationFunctionWithExtraDeps() throws Exception {
-        testExclamationFunction(Runtime.PYTHON, false, false, true);
+        testExclamationFunction(Runtime.PYTHON, false, false, false, true);
     }
 
     @Test(groups = {"python_function", "function"})
     public void testPythonExclamationZipFunction() throws Exception {
-        testExclamationFunction(Runtime.PYTHON, false, true, false);
+        testExclamationFunction(Runtime.PYTHON, false, true, false, false);
     }
 
     @Test(groups = {"python_function", "function"})
     public void testPythonExclamationTopicPatternFunction() throws Exception {
-        testExclamationFunction(Runtime.PYTHON, true, false, false);
+        testExclamationFunction(Runtime.PYTHON, true, false, false, false);
     }
 
     @Test(groups = {"python_function", "function"})

From fb9c4d034bfdcfdf256720ec2490c737cae53839 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Wed, 10 May 2023 21:16:17 +0800
Subject: [PATCH 266/404] [fix][fn] Make pulsar-admin support update py/go with
 package url (#19897)

---
 .../main/java/org/apache/pulsar/admin/cli/CmdFunctions.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 91ec7183ff13c..9b30d59f1679c 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -1037,6 +1037,10 @@ void runCmd() throws Exception {
             updateOptions.setUpdateAuthData(updateAuthData);
             if (Utils.isFunctionPackageUrlSupported(functionConfig.getJar())) {
                 getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getJar(), updateOptions);
+            } else if (Utils.isFunctionPackageUrlSupported(functionConfig.getPy())) {
+                getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getPy(), updateOptions);
+            } else if (Utils.isFunctionPackageUrlSupported(functionConfig.getGo())) {
+                getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getGo(), updateOptions);
             } else {
                 getAdmin().functions().updateFunction(functionConfig, userCodeFile, updateOptions);
             }

From 96367e1c2bfd64ec0f72602321ae6ede4bed96cd Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 10 May 2023 09:52:45 -0500
Subject: [PATCH 267/404] [cleanup] Deduplicate test certificates to simplify
 management (#20289)

---
 .../broker/admin/AdminApiTlsAuthTest.java     | 38 +++++++++----------
 .../admin/BrokerAdminClientTlsAuthTest.java   | 18 ++++-----
 .../auth/MockedPulsarServiceBaseTest.java     | 15 ++++++++
 .../authentication/tls-http/admin.cert.pem    | 26 -------------
 .../authentication/tls-http/admin.key-pk8.pem | 28 --------------
 .../authentication/tls-http/broker.cert.pem   | 27 -------------
 .../tls-http/broker.key-pk8.pem               | 28 --------------
 .../authentication/tls-http/ca.cert.pem       | 29 --------------
 .../authentication/tls-http/proxy.cert.pem    | 26 -------------
 .../authentication/tls-http/proxy.key-pk8.pem | 28 --------------
 .../tls-http/superproxy.cert.pem              | 26 -------------
 .../tls-http/superproxy.key-pk8.pem           | 28 --------------
 .../authentication/tls-http/user1.cert.pem    | 26 -------------
 .../authentication/tls-http/user1.key-pk8.pem | 28 --------------
 .../server/AuthedAdminProxyHandlerTest.java   | 32 +++++++---------
 .../SuperUserAuthedAdminProxyHandlerTest.java | 26 ++++++-------
 .../tls-admin-proxy/admin.cert.pem            | 26 -------------
 .../tls-admin-proxy/admin.key-pk8.pem         | 28 --------------
 .../tls-admin-proxy/broker.cert.pem           | 27 -------------
 .../tls-admin-proxy/broker.key-pk8.pem        | 28 --------------
 .../tls-admin-proxy/ca.cert.pem               | 29 --------------
 .../tls-admin-proxy/proxy.cert.pem            | 26 -------------
 .../tls-admin-proxy/proxy.key-pk8.pem         | 28 --------------
 .../tls-admin-proxy/randouser.cert.pem        | 19 ----------
 .../tls-admin-proxy/randouser.key-pk8.pem     | 28 --------------
 .../tls-admin-proxy/superproxy.cert.pem       | 26 -------------
 .../tls-admin-proxy/superproxy.key-pk8.pem    | 28 --------------
 .../tls-admin-proxy/user1.cert.pem            | 26 -------------
 .../tls-admin-proxy/user1.key-pk8.pem         | 28 --------------
 29 files changed, 64 insertions(+), 712 deletions(-)
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem
 delete mode 100644 pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java
index bd23e0d7e4e85..2b7b9101a81f2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java
@@ -65,19 +65,15 @@
 @Test(groups = "broker-admin")
 public class AdminApiTlsAuthTest extends MockedPulsarServiceBaseTest {
 
-    private static String getTLSFile(String name) {
-        return String.format("./src/test/resources/authentication/tls-http/%s.pem", name);
-    }
-
     @BeforeMethod
     @Override
     public void setup() throws Exception {
         conf.setLoadBalancerEnabled(true);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(getTLSFile("broker.cert"));
-        conf.setTlsKeyFilePath(getTLSFile("broker.key-pk8"));
-        conf.setTlsTrustCertsFilePath(getTLSFile("ca.cert"));
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setAuthenticationEnabled(true);
         conf.setAuthenticationProviders(
                 Set.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls"));
@@ -87,8 +83,8 @@ public void setup() throws Exception {
 
         conf.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.AuthenticationTls");
         conf.setBrokerClientAuthenticationParameters(
-                String.format("tlsCertFile:%s,tlsKeyFile:%s", getTLSFile("admin.cert"), getTLSFile("admin.key-pk8")));
-        conf.setBrokerClientTrustCertsFilePath(getTLSFile("ca.cert"));
+                String.format("tlsCertFile:%s,tlsKeyFile:%s", getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8")));
+        conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setBrokerClientTlsEnabled(true);
         conf.setNumExecutorThreadPoolSize(5);
 
@@ -115,11 +111,11 @@ WebTarget buildWebClient(String user) throws Exception {
             .register(JacksonConfigurator.class).register(JacksonFeature.class);
 
         X509Certificate trustCertificates[] = SecurityUtility.loadCertificatesFromPemFile(
-                getTLSFile("ca.cert"));
+                CA_CERT_FILE_PATH);
         SSLContext sslCtx = SecurityUtility.createSslContext(
                 false, trustCertificates,
-                SecurityUtility.loadCertificatesFromPemFile(getTLSFile(user + ".cert")),
-                SecurityUtility.loadPrivateKeyFromPemFile(getTLSFile(user + ".key-pk8")));
+                SecurityUtility.loadCertificatesFromPemFile(getTlsFileForClient(user + ".cert")),
+                SecurityUtility.loadPrivateKeyFromPemFile(getTlsFileForClient(user + ".key-pk8")));
         clientBuilder.sslContext(sslCtx).hostnameVerifier(NoopHostnameVerifier.INSTANCE);
         Client client = clientBuilder.build();
 
@@ -133,8 +129,8 @@ PulsarAdmin buildAdminClient(String user) throws Exception {
             .serviceHttpUrl(brokerUrlTls.toString())
             .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls",
                             String.format("tlsCertFile:%s,tlsKeyFile:%s",
-                                          getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8")))
-            .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build();
+                                          getTlsFileForClient(user + ".cert"), getTlsFileForClient(user + ".key-pk8")))
+            .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
     }
 
     PulsarClient buildClient(String user) throws Exception {
@@ -143,8 +139,8 @@ PulsarClient buildClient(String user) throws Exception {
             .enableTlsHostnameVerification(false)
             .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls",
                             String.format("tlsCertFile:%s,tlsKeyFile:%s",
-                                          getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8")))
-            .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build();
+                                          getTlsFileForClient(user + ".cert"), getTlsFileForClient(user + ".key-pk8")))
+            .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
     }
 
     @Test
@@ -471,11 +467,11 @@ public void testDeleteNamespace() throws Exception {
     public void testCertRefreshForPulsarAdmin() throws Exception {
         String adminUser = "admin";
         String user2 = "user1";
-        File keyFile = new File(getTLSFile("temp" + ".key-pk8"));
+        File keyFile = File.createTempFile("temp", ".key-pk8");
         Path keyFilePath = Paths.get(keyFile.getAbsolutePath());
         int autoCertRefreshTimeSec = 1;
         try {
-            Files.copy(Paths.get(getTLSFile(user2 + ".key-pk8")), keyFilePath, StandardCopyOption.REPLACE_EXISTING);
+            Files.copy(Paths.get(getTlsFileForClient(user2 + ".key-pk8")), keyFilePath, StandardCopyOption.REPLACE_EXISTING);
             PulsarAdmin admin = PulsarAdmin.builder()
                     .allowTlsInsecureConnection(false)
                     .enableTlsHostnameVerification(false)
@@ -483,8 +479,8 @@ public void testCertRefreshForPulsarAdmin() throws Exception {
                     .autoCertRefreshTime(autoCertRefreshTimeSec, TimeUnit.SECONDS)
                     .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls",
                                     String.format("tlsCertFile:%s,tlsKeyFile:%s",
-                                                  getTLSFile(adminUser + ".cert"), keyFile))
-                    .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build();
+                                                  getTlsFileForClient(adminUser + ".cert"), keyFile))
+                    .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
             // try to call admin-api which should fail due to incorrect key-cert
             try {
                 admin.tenants().createTenant("tenantX",
@@ -496,7 +492,7 @@ public void testCertRefreshForPulsarAdmin() throws Exception {
             // replace correct key file
             Files.delete(keyFile.toPath());
             Thread.sleep(2 * autoCertRefreshTimeSec * 1000);
-            Files.copy(Paths.get(getTLSFile(adminUser + ".key-pk8")), keyFilePath);
+            Files.copy(Paths.get(getTlsFileForClient(adminUser + ".key-pk8")), keyFilePath);
             MutableBoolean success = new MutableBoolean(false);
             retryStrategically((test) -> {
                 try {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java
index 54164c3d40ef5..19a550457a4dd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java
@@ -48,10 +48,6 @@ public void beforeMethod(Method m) throws Exception {
         methodName = m.getName();
     }
 
-    private static String getTLSFile(String name) {
-        return String.format("./src/test/resources/authentication/tls-http/%s.pem", name);
-    }
-
     @BeforeMethod
     @Override
     public void setup() throws Exception {
@@ -63,19 +59,19 @@ public void setup() throws Exception {
 
     private void buildConf(ServiceConfiguration conf) {
         conf.setLoadBalancerEnabled(true);
-        conf.setTlsCertificateFilePath(getTLSFile("broker.cert"));
-        conf.setTlsKeyFilePath(getTLSFile("broker.key-pk8"));
-        conf.setTlsTrustCertsFilePath(getTLSFile("ca.cert"));
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setAuthenticationEnabled(true);
         conf.setSuperUserRoles(Set.of("superproxy", "broker.pulsar.apache.org"));
         conf.setAuthenticationProviders(
                 Set.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls"));
         conf.setAuthorizationEnabled(true);
         conf.setBrokerClientTlsEnabled(true);
-        String str = String.format("tlsCertFile:%s,tlsKeyFile:%s", getTLSFile("broker.cert"), getTLSFile("broker.key-pk8"));
+        String str = String.format("tlsCertFile:%s,tlsKeyFile:%s", BROKER_CERT_FILE_PATH, BROKER_KEY_FILE_PATH);
         conf.setBrokerClientAuthenticationParameters(str);
         conf.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.AuthenticationTls");
-        conf.setBrokerClientTrustCertsFilePath(getTLSFile("ca.cert"));
+        conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setTlsAllowInsecureConnection(true);
         conf.setNumExecutorThreadPoolSize(5);
     }
@@ -93,8 +89,8 @@ PulsarAdmin buildAdminClient(String user) throws Exception {
             .serviceHttpUrl(brokerUrlTls.toString())
             .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls",
                             String.format("tlsCertFile:%s,tlsKeyFile:%s",
-                                          getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8")))
-            .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build();
+                                          getTlsFileForClient(user + ".cert"), getTlsFileForClient(user + ".key-pk8")))
+            .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
     }
 
     /**
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
index 3fe8c22b1c4de..b688d5fbf24d8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
@@ -66,6 +66,21 @@
  * Base class for all tests that need a Pulsar instance without a ZK and BK cluster.
  */
 public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport {
+    // All certificate-authority files are copied from the tests/certificate-authority directory and all share the same
+    // root CA.
+    protected static String getTlsFileForClient(String name) {
+        return ResourceUtils.getAbsolutePath(String.format("certificate-authority/client-keys/%s.pem", name));
+    }
+    public final static String CA_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
+    public final static String BROKER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    public final static String BROKER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    public final static String PROXY_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/proxy.cert.pem");
+    public final static String PROXY_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/proxy.key-pk8.pem");
     public final static String BROKER_KEYSTORE_FILE_PATH =
             ResourceUtils.getAbsolutePath("certificate-authority/jks/broker.keystore.jks");
     public final static String BROKER_TRUSTSTORE_FILE_PATH =
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem
deleted file mode 100644
index 0665edbdc126c..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/admin.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEZTCCAk2gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyMjA4NTcwNloYDzIyOTIwNDA2MDg1NzA2WjAQMQ4wDAYDVQQD
-DAVhZG1pbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJw3Jfbn0xkW
-36kqQjES6Hn+YTZ2jXS5Co2MzsGBsIY0qJ2BbWHSSaMrja4IERUaCQp16SWxPmZ0
-srMm6ErDoap+O70CWXLT3ybYMV5aVwv3ca4uxsedzaw9MpFXfUDsJJ3yre1JpO+t
-A/QzJEGq1d6NN49InUP5kB1Rpay3vaxx8hduzqTO+E/Lptv92p6GjOpXi2icSjiA
-pgaan2ldGGKEKv2Sc2bfdIDkTq1yDyNmuPET0yD2dci106EW/mPyj81umPKG/o4K
-5W18yG/IhXw5W1zlgO1fWCuqva8NCBdu7s1c7hUX8DBx7km4/I7dllz/nYHIfCEQ
-Dmj38oQjYk8CAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
-oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
-ZmljYXRlMB0GA1UdDgQWBBQTMzAAOJ9gXvQSS7Be3+qmrb1kVDAfBgNVHSMEGDAW
-gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
-FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQAwy8f6hsG0
-85e3SOIztbUnaVxS7wzDeDzR3vCjpXpm4vTToYzN9zx3JHKSdJrB12emVxItwW/7
-bXqBk0n2EdQjRHCuebnY05eFMNGagMEEMVmSLOproQD7VsyALNxCss1JAyRikh70
-W7wgOVeAhqE53UqqrkzTE7Q+8Bag9t3FytHxApY17XglbWkiVcFpQwSURe9Emi3E
-aCJCryGJXrBNuCFXGzetSygDEy27+2FeH8S2XsMwUEGLqDDehzvMenVz1xjXtq+s
-KPkofAde52NHd4lLkSeBMSFnKe3V7Xxax2OEUsoQRF3bkbpcJSWsKS9ZAA2yrtuy
-Nz/aA1F42LuSFPAYQr1kcZ8eSS918RWz+BiJYU2JuUOPd1XUmJXVvZ4CJurWaC7+
-ZD51YdD8E245xd55fsA6/qLx3eE/Kp0dVq+Hxuz6b4yLET0zkGunOe4A3hnRgkOA
-XolXCL+VthhWtFGXn8CjpxDnzjahq69Io+dINehqd5aJEgvnHZIK2s7FTqqBBodU
-HhyAE94f64z7ziuRhEG54bmBF+MoGyPf6dVn1Mp3+o+YeQ5q6XlKgh+u8jMgmqRO
-ikdsVdMqopt/FXh9eFzvQrwOZFLK6JE/edUgb6xvS1jMF5zi2lIlIkq1RBQOr4HT
-XDwX4vRtfpDxpsetGVpeq9O07fbMvp+mkw==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem
deleted file mode 100644
index 6aaa22c44d746..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/admin.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCcNyX259MZFt+p
-KkIxEuh5/mE2do10uQqNjM7BgbCGNKidgW1h0kmjK42uCBEVGgkKdeklsT5mdLKz
-JuhKw6Gqfju9Ally098m2DFeWlcL93GuLsbHnc2sPTKRV31A7CSd8q3tSaTvrQP0
-MyRBqtXejTePSJ1D+ZAdUaWst72scfIXbs6kzvhPy6bb/dqehozqV4tonEo4gKYG
-mp9pXRhihCr9knNm33SA5E6tcg8jZrjxE9Mg9nXItdOhFv5j8o/Nbpjyhv6OCuVt
-fMhvyIV8OVtc5YDtX1grqr2vDQgXbu7NXO4VF/Awce5JuPyO3ZZc/52ByHwhEA5o
-9/KEI2JPAgMBAAECggEAP+Ipq2Q4puz8wGBgu1LhMWp+9Nfcl1xI3YQ01VulBe0o
-+2h/g96McKcSBJaV7cw84ENB+kEWpK2amrsRiemhBmkjIvOAAv50Jp2I6u4E5Qbn
-PXUxo1Z8UrCgKmHd/hvUCafByuUwBzf5AvebHyOu3JlhnD302mSHtAW8u/pUHd3D
-sxJaw1zwQvmlD5ryM2IVYSji7NYCXF0H6V7HfyohTCrQFEWEAdqEDcFR1BUwbPCE
-raq7sAiEy+cBUnfV3IOEAffOZy0vSR90/WwERcwrzCdZmpWpTqtbcqtdBPqsSQzX
-shDvXd0e43+FSJzCtQsSQ8WzIrp3rgKJUDA1pJQW4QKBgQDJdTcB6qZz8r+4q9gc
-q1KAJyMy01Vio1yaqYzXr0C9Z5FW1GhL+4fwer2y9JyD45sb/lP4reFj9S193BNR
-C8cdxM5GrWEpzaQ0Dt1s8P1UbU8G6r4NqwI6ORF3CxXZKfXivQcgqBurJGrBdjIC
-NwqAzSkX5flBbhfTlJuUH87k3wKBgQDGgjzdIWXab7FZfdUzzrtEwNooBiSEFixm
-UAwP5sxL8VkM9wzAKPEVDDQBBoKIga9OESif4S9UUo4tu65AYfxF9Om26K4QrVj8
-HT/U+lfT7xFmPd/GINIbeTSmW0w7Ehpj8SbcQI4Sb2lVE562FlHh7QbHZd0/X/2J
-nbgT9MRAkQKBgQCVPAN3o/+SPOzRPFtnQXJoBJYKfIrv+twKpjbzP5vRsvrzO33X
-a4kUF5iXDKU0/lJUtl42BXjFt0Xvyit1CiiCYNv9d0pW0UMmXSyiGxNOi3rTQOlw
-7pFD2Cqb6NZSfMbtI+I3ytBUQzHiBlCdW3CoYVJjpbSzR37W+WsWm0mEOQKBgQCq
-ANObFYUjA1DBMZCrY7rhcL/kUw5myI6RuK/71k7UIwd+oP0cfHOq8N6AmlCkE1xM
-4UkHU1SzRFhbNkZPARuJ1etqJ+8afTqd/3axMQyShkVCaG8CQQ1vVegPKFUqqaBM
-QzRioC6L/zoYEEt16buKXvHVRpmqMszxVE9XV+HS4QKBgDvs5qloOowS5kcWInrj
-yecu5MJvFf2IZpMw7EiKV8VUPeKaiUlqgFj9d9cotUIMauXgBq6f5NBRg7Ike60t
-/JJrPtqXY+gdFLjxcKMUVYhomFlQYYg/RJZUBrtkyKBP68abopCYmb59r3ixeNNf
-qA1F36mmFtzdjSdtH/dTTecN
------END PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem
deleted file mode 100644
index b5c7a5dc709a1..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/broker.cert.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD
-DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
-DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63
-EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O
-OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01
-+ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X
-aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s
-bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw
-ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
-ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA
-DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx
-DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV
-HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ
-ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y
-ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN
-zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9
-spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur
-x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0
-fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d
-mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS
-efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp
-0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C
-cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK
-3N2cGJWRyvpvvmbP2hKCa4eLgEc=
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem
deleted file mode 100644
index 2b51d015b8ace..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/broker.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq
-mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM
-6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU
-TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh
-QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ
-M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN
-95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j
-WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN
-wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y
-3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK
-r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv
-yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP
-eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj
-bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF
-M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP
-PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9
-T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2
-yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP
-UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat
-V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt
-6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm
-3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ
-2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA
-G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+
-I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI
-5qJ7jxgksHWji4lYfL4r4J6Zaw==
------END PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem
deleted file mode 100644
index 0446700135d39..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/ca.cert.pem
+++ /dev/null
@@ -1,29 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFCDCCAvCgAwIBAgIJANfih0+geeIMMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV
-BAMMBmZvb2JhcjAeFw0xODA2MjIwODQ2MjFaFw0zODA2MTcwODQ2MjFaMBExDzAN
-BgNVBAMMBmZvb2JhcjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOVU
-UpTPeXCeyfUiQS824l9s9krZd4R6TA4D97eQ9EWm2D7ppV4gPApHO8j5f+joo/b6
-Iso4aFlHpJ8VV2a5Ol7rjQw43MJHaBgwDxB1XWgsNdfoI7ebtp/BWg2nM3r8wm+Z
-gKenf9d1/1Ol+6yFUehkLkIXUvldiVegmmje8FnwhcDNE1eTrh66XqSJXEXqgBKu
-NqsoYcVak72OyOO1/N8CESoSdyBkbSiH5vJyo0AUCjn7tULga7fxojmqBZDog9Pg
-e5Fi/hbCrdinbxBrMgIxQ7wqXw2sw6iOWu4FU8Ih/CuF4xaQy2YP7MEk4Ff0LCY0
-KMhFMWU7550r/fz/C2l7fKhREyCQPa/bVE+dfxgZ/gCZ+p7vQ154hCCjpd+5bECv
-SN1bcVIPG6ngQu4vMXa7QRBi/Od40jSVGVJXYY6kXvrYatad7035w2GGGGkvMsQm
-y53yh4tqQfH7ulHqB0J5LebTQRp6nRizWigVCLjNkxJYI+Dj51qvT1zdyWEegKr1
-CthBfYzXlfjeH3xri1f0UABeC12n24Wkacd9af7zs7S3rYntEK444w/3fB0F62Lh
-SESfMLAmUH0dF5plRShrFUXz23nUeS8EYgWmnGkpf/HDzB67vdfAK0tfJEtmmY78
-q06OSgMr+AOOqaomh4Ez2ZQG592bS71G8MrE7r2/AgMBAAGjYzBhMB0GA1UdDgQW
-BBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAfBgNVHSMEGDAWgBRXC+nLI+i/Rz5Qej9F
-fqEYQ50VJzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
-9w0BAQsFAAOCAgEAYd2PxdV+YOaWcmMG1fK7CGwSzDOGsgC7hi4gWPiNsVbz6fwQ
-m5Ac7Zw76dzin8gzOPKST7B8WIoc7ZWrMnyh3G6A3u29Ec8iWahqGa91NPA3bOIl
-0ldXnXfa416+JL/Q5utpiV6W2XDaB53v9GqpMk4rOTS9kCFOiuH5ZU8P69jp9mq6
-7pI/+hWFr+21ibmXH6ANxRLd/5+AqojRUYowAu2997Z+xmbpwx/2Svciq3LNY/Vz
-s9DudUHCBHj/DPgNxsEUt8QNohjQkRbFTY0a1aXodJ/pm0Ehk2kf9KwYYYduR7ak
-6UmPIPrZg6FePNahxwMZ0RtgX7EXmpiiIH1q9BsulddWkrFQclevsWO3ONQVrDs2
-gwY0HQuCRCJ+xgS2cyGiGohW5MkIsg1aI0i0j5GIUSppCIYgirAGCairARbCjhcx
-pbMe8RTuBhCqO3R2wZ0wXu7P7/ArI/Ltm1dU6IeHUAUmeneVj5ie0SdA19mHTS2o
-lG77N0jy6eq2zyEwJE6tuS/tyP1xrxdzXCYY7f6X9aNfsuPVQTcnrFajvDv8R6uD
-YnRStVCdS6fZEP0JzsLrqp9bgLIRRsiqsVVBCgJdK1I/X59qk2EyCLXWSgk8T9XZ
-iux8LlPpskt30YYt1KhlWB9zVz7k0uYAwits5foU6RfCRDPAyOa1q/QOXk0=
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem
deleted file mode 100644
index 6c2f4295c9dbd..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEZTCCAk2gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyNTEzNTYwNFoYDzIyOTIwNDA5MTM1NjA0WjAQMQ4wDAYDVQQD
-DAVwcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMieX78Yj8OW
-eBHAneRaCCoO8Qrpj8zGo7h9lCdmi1lBDh1uR2sDbotiHGfJzQn836WcYmyeAvfn
-qvgr9HCXmXdLgmJ3GT/LVu5GEm6msSDiZQPr9so5lQVioisK4UwJROQsE/J52cyR
-9o3H6M4FKb6QpoobKa62fSfTumwwulaYaDJuRRGoGIkcRuUQ59EWAaDkD3IcDpAn
-9mTbnE4Iz+JxSrsZ5DJ3X/m/AqyLWtj6GAfyK9a1dhNdlf2x4JZT1QNtojiBXt95
-OIZyRBNbHMFniq5gel6wdBkmJWutfcTct7wKa2LCxLpKoDIc1HWoL3+RUzOKxYIP
-0qXEQ3bmONkCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
-oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
-ZmljYXRlMB0GA1UdDgQWBBSgsgfmDbXEkrrpHUC9GnDDjxaKizAfBgNVHSMEGDAW
-gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
-FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQB+uZ2OWR+G
-sRqYHEeVqUI8G2p8Np8eC/onGpBL8Gj9SGxIQcIPtqHPUlCe9fd/96JOptOGRYEB
-BmUCaCmQ4IgMW6e6fArka5IB4XXIgHFXyQ6ImTvjDavzlVw06zn9S4dLwVzsRBg+
-GS9svtq23W+f5rEN5N+7LhtcbclfiG4VCCqDG5VhkzEok+SRamDI8rDRZtodMw0O
-/+L+xaaQPUPjX8KUlKn4uVpCDbxUzHonlCPzbkHHm5su0D4ysjJIy3/y3yow6JE/
-02L7PZkmkmw3/V+84T3X8/GD15sVUv/3v1gXEBxYwAs+RNTJ0APvMEMSvCq0AMfF
-bPMZBuAGNBG7lv7TovzHgGFKXT7du5OFF/qjAsEffhbo224CB96fgwvvndwHHBFh
-J06BvHZG1i9dDVhUKoB1owkWrE4RZv2ZKEtZYgizzSmzZRHtARo0t1Byc5djx1tX
-TkJOHshNqJZOY1ER0DPaVQgKI+PRTbEdj/xPGRX3ebSqDmilAfPXshqgElfch6Yl
-f2V58TyCnjXOibvkG9D5OyCdWLEECumOZgYar0KZgNfrvTOi1OKXnX1fsbh29fWA
-ICZRcdmjkz79zQXY2SuzCWlskuXPKAmW1AMqs+l6ormmKfzIUx3Yriy4LqIIYY1v
-uQD5vghZmd9HUg2KaXfSGD9stGCD8KhntA==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem
deleted file mode 100644
index 70e0107f2741f..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/proxy.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDInl+/GI/DlngR
-wJ3kWggqDvEK6Y/MxqO4fZQnZotZQQ4dbkdrA26LYhxnyc0J/N+lnGJsngL356r4
-K/Rwl5l3S4Jidxk/y1buRhJuprEg4mUD6/bKOZUFYqIrCuFMCUTkLBPyednMkfaN
-x+jOBSm+kKaKGymutn0n07psMLpWmGgybkURqBiJHEblEOfRFgGg5A9yHA6QJ/Zk
-25xOCM/icUq7GeQyd1/5vwKsi1rY+hgH8ivWtXYTXZX9seCWU9UDbaI4gV7feTiG
-ckQTWxzBZ4quYHpesHQZJiVrrX3E3Le8CmtiwsS6SqAyHNR1qC9/kVMzisWCD9Kl
-xEN25jjZAgMBAAECggEAJJyCbKVW1yLGlrbIGbw0cTh41Lz6+SvnBOwl9WrJU2iD
-4usVLXpa2iT1ehthx8jWJ6r6a0gK0qL8mH2tBj8kSpkFGmMRwIqjOqifBIJ3IMEw
-Hh8Z0p3fjDQL1D8QDohCgkFpAn8qOCMLE6S/35khnR1Yxytd1/yFqpcBFm1uFA85
-dYisjPqWm/IZIU5rH0zgKAIhtvl9abnoi93443EHsKpRAW1gwRXx9Aak7TV768bZ
-tELBsaTnXnNzamDiaimmxEOlqR9O0W8JE/31KFL26JcVmsTRG7sMpoUxCEMjOuGZ
-J30bXFZUW6NrDpFsQ7uTqD6TNn2971N8KFCLnC/JYQKBgQDo82axC7L7n7BYTu18
-dupeT7n5dTBD/I3l0KtT05xiZA8GZr2i+pt+/aWzCzK4/Ee4jb4/o8CRQRB5v5mo
-c9lc+BaoAIQiwiw+aufT+UojrcijrOMEL5Zk3zdZ2rcEoAsVvqtejNnwLCGI9Rnl
-gp7n9oRhwDIv9Fu09snUougE3wKBgQDceAGKUB8pGd3eEya/0jU9J60LsKbcJSsN
-4v1S5LiPtHOyhr0g4x/LibMP2PJhG3tJ1bgpaGmn9du2D20M6ukRhIYyn/7G+N+A
-oqryyvO1MMYnhc4IEQvWrzDnBM0hV2bdjp4s/1ASVHVRwk8+orqxysIJ1D75nnRX
-Tyfl6HgBRwKBgQCfVlWIhiMPv6OkU6BXgRNAHTJs8f5okmgQqNF3jgequRwZ2c6e
-muIfU6myNNel9lGsZ6+Y4g4GjMWTMT4OHeewkrUUhv3atIwEyaT2tc5DZ0wUwF2r
-cE1jg9bdbB/BVyMd5YRcMOWlRNpPTq8+8EB3E4RrREZPzMmplyBohGFFawKBgGjc
-P0dM8nU3E1rj2wNTdPUAYQL1Y3fDyeWR+BEsLkhTeNAJ2/y/akkB1oQMGMRtMMee
-ejhfrBkyC+1dCu4g8PffA4EirihvCMcDF7HhK+cbKrRzpNobWXkj3GuU0ggwrQFm
-Kv+V87y0JRTdCZnuBkQ3/vBz3fwWDJnWUVC9sA5TAoGBAN4t2gJCZax5yInUgyWi
-Tgumb2qVWsGBBLMTKIsrkK/KphzgAhHcVhDCybA79TmIM2FfIlpf6TE7Mv4NI055
-ZJzHX+GMT5Czy2Ku9MJD3PLTFN5MjYb6g92fKViLDMI6fwzTHB8xPJ7Ob9bV2srS
-ZlmKNXTkZFk3/orK4WdCZufz
------END PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem
deleted file mode 100644
index 9656e2c8ba0d2..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEajCCAlKgAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyNjEzMjUxN1oYDzIyOTIwNDEwMTMyNTE3WjAVMRMwEQYDVQQD
-DApzdXBlcnByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA43Sn
-ys/h3wxJ/IBEzbJdQAKCxU4of5KgTDzFoOaS8C63Nwbjgy0qcWYdRDceP4lJIKSA
-1+JZ+R1opyrfVIC2D9oDJFIJTFfXy9G9VYDccwAONPgAamvRzBnYcEu8fqM+Ohle
-kZltKktAHnX3WtG3RyxEL5nYzlMlGwUXJu3Rxc9SlkYSxERzjWHikGqGmXLX2qB0
-k6oyxTrK+4+EHk3khbEIqyQZSOFbD7NMfnRy0CWFv/9T1shgjAIBCake6jY7lwaT
-S7JvbLfG6ABf5xHMxoWLXa2qwb+Ar43Ff9g8kKZwFOxMeGcwkzyJPBbWytFxaWn+
-R2RHhTaVCGc22CjdfQIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB
-BAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg
-Q2VydGlmaWNhdGUwHQYDVR0OBBYEFHEgLhfH0Z1QlLxeQbG7YZyBlz1MMB8GA1Ud
-IwQYMBaAFFcL6csj6L9HPlB6P0V+oRhDnRUnMA4GA1UdDwEB/wQEAwIF4DAdBgNV
-HSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBAK9q
-3YnEa99Cq3Vo3g6PE/A/xOE97seVNuavqyBcVv9PrTydb5XG4jPkYU3xOYXIc8rA
-A4gzd+AsGO9rjGMPGGjQJI3JO/3BCeBJMkn50C/rM1yWVMHnVyFcJlg16xaWtj1e
-2Jk8egJW2gSYyF+N2TdzI7tOb002GNr36posnqO+IOoLapyHFBxxUjsPDRoo8fJn
-myWsV1Y9oRUZyJlfIAJsu85ew7gDBY2jaiEiopzour3uU3C0N7gYni2OmVwfr6J8
-R2/Jp43BSD5sYOW9RAJIEEXef+InYtz9HTJvKu2LsWwIBkaztk29tJcDE+1La6Sw
-0dF0YkUwnXoGQFjiV+8pXX3TF5glXKj1rU8WfNazF6lqslB6DmdgR3/FQ6Z2sE86
-d9hVtayZIGlzU0rWmBBtr++7Wo88nmzAtd/xbZMFG8U//+Q2AvJT2oVGtqM48+al
-rnsN/gYrLDr7RC14bHIuO1v6ZL/rAi7SPKrKYAyQVTAcRuW516SxxR6S1Xa1ITnh
-rwgKg13eQuwu3iigguIS9XL6nAXabBIxBxMl6o2YlyIPekKYIcQmpqhkavJ6VOgX
-iq9VdY6fIJVfmxNZuwM3/28k7UeUAfhI2SSVH4ZURbPiGGH2wukc1QkmtZ2cNa35
-C1y79aqJbIa3ErqLFPj/fM+34x8L7QHPq6RfaODa
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem
deleted file mode 100644
index 2e4140b8fb392..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/superproxy.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDjdKfKz+HfDEn8
-gETNsl1AAoLFTih/kqBMPMWg5pLwLrc3BuODLSpxZh1ENx4/iUkgpIDX4ln5HWin
-Kt9UgLYP2gMkUglMV9fL0b1VgNxzAA40+ABqa9HMGdhwS7x+oz46GV6RmW0qS0Ae
-dfda0bdHLEQvmdjOUyUbBRcm7dHFz1KWRhLERHONYeKQaoaZctfaoHSTqjLFOsr7
-j4QeTeSFsQirJBlI4VsPs0x+dHLQJYW//1PWyGCMAgEJqR7qNjuXBpNLsm9st8bo
-AF/nEczGhYtdrarBv4CvjcV/2DyQpnAU7Ex4ZzCTPIk8FtbK0XFpaf5HZEeFNpUI
-ZzbYKN19AgMBAAECggEBAM1JAwuD1drmj3wKFI8F1S2pVndXFCwXnP9RthiDIckO
-kKNkX0CMKgtQ20cu6+jyMgL5FaRCkWvJxCNkCU6OIENsQ3urYuL5QTWeZeBevhg4
-y5m43z8tcptgFD09zbEKCmaLcRO9wo3yfrs/QvE/58efxyajFs8YsZuSW5Px/msl
-AFN8qGY0q0lQbQPK8cPYT4OnW9isqEjkvHq1Q+ryv4cxsGWtBZYmJVeLVHzClihi
-lODdN6YsDcu9jLwsA8o2WSBRsbLHK6caW4FdEwgOLckc0EGGEU64VEB9ZOwgcwYQ
-M9mZDs2w7qk63YnDH4KmpoP0Oc8N+bG7Pkifd7f8HqECgYEA8esMQya6Qln4O7Qo
-aI8X9t5gCpq+bh/P+7vYdBR/9St5VtE/P0yuaQdT2e5t+Ei4ujqHvJ0GTPmqYIJf
-2DNvJMu8EnXv+nqKQCsyMc/nqQN1CcRGqZssH1V/ZIQLRUA0cN4hzE0eHITwp6U9
-vD/WaE3mKX/XHthIjc8hnuoajOkCgYEA8LIYMgyD3wClWOKe5TkBumI0KvA9tP90
-IFHu1wLNl0tKBpsXkzzxiav1FMX3K7B29vxukrs946KRESHzRg4c5ULPnMmwcQyx
-AortKJWrGVsna/QDWPRutXSN1XmnjKWsHmTpQQqLfaRvLW+Hd34+EVdVh9sVt/y9
-RucnBvxcX3UCgYBebm/U7pMaP2BkfcigN+sU1G0M9qaK+iQHkaXGehIQs62js/5K
-STZzjQawNR/8IPbqytodR/YjqflVvs6G6FzkMhrx4dORJLA+qB3pz8wP72eKLnGe
-1xF8EbWumNSFbbCKtkrfIuM0IriF2Dym9QxOnsnPPTXNtoNrx4TKMXu3sQKBgBq9
-noSI8WmoF7adTsvmnnOHj4YptKFUNCGXGLLYg+DII4xCVMct4SPLb+oD6Gb5Lu5X
-sy0oEkMk/3roy68/yCQMXSZtHeYhY9UFfD2jCyRBBUswC+MpHNeaAFv0LRIqIcoq
-qeNo+YBW8WcZ2fIDm3+vtTfntiz/rkOfUK2tAdI1AoGAL2LVDF5e1meSRocEoy8e
-gqrshrhg+KBqYmcjtd4Iup4WvR6uyH4qE9yFLLHFZ04pZzXLHcPfqgOSP8qTxgH3
-h0uqtcYmejS/yl6PbC9OPXcSkMgntRkTbU9Ug05ijfN9NRnW+8WXvi8Z6DKed1P1
-9PxwA75P9o0h00mMk8PQitQ=
------END PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem b/pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem
deleted file mode 100644
index 072f2867fe62b..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/user1.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEZTCCAk2gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyNzA4NDAyNVoYDzIyOTIwNDExMDg0MDI1WjAQMQ4wDAYDVQQD
-DAV1c2VyMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM5iqgr4PUUZ
-AW9MDGP5cBaSJALSPV63m6M/IoovrMWJ9CGtcQfZTUHwDorIlXgQ6H/KufmsHW0Y
-OQbChLSTDB14D0jSMtyv6+ibSoE1ZEl2SbB1miLd0P5AS9YmzzEW2+bx0zJORLYD
-PzJ1Nh3/kQlRs04IECki291WZiVRzX2JRoL7kMtOAoKJqQfsT14Oi9EAw39VhLeB
-uc/Mx6Jsutq/YdXakoZtQbfZka2MMfLXgMDLIPDqbU+09q7au2dq8RjGzrWnxnOX
-o/XQssrIbwzJJYASBsgAAtnAw7bPzCX6+cL6PZVRyiEZov0HKXyRyvrbQ5hyEMuS
-3dHqoKt0fKMCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
-oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
-ZmljYXRlMB0GA1UdDgQWBBQ7NSD6lx6Vq38cEoD5l7FHs1Ej8DAfBgNVHSMEGDAW
-gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
-FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQC+cU7ctY7o
-eTW+oHTq9EGJUMwW1fOww4QDrHtgZT4OYkO88zxQV2Cr050p8eaV5dHXZBf9/bRV
-7hPNV5+HpQhb9TZ9xK2WRZ2QV7a/UDyUnksVKGSK9tNZMZPueOEB19e4bIBcgnQa
-5i9sgZr93na7pFOY7lBQy6gfaOcnejYHmvVqIGaZBVH8rkEsGhhkJxy7qkpFNKgf
-PGiIRo9L0WYqDCSiaICeCiteJwIfjsUFJKF0YnpXZq1kFfQscnleg60MWZAXvacp
-tAciE51Ow60cqQWER66iwqnBSPD4l91SxAaGQAmalgCioGsYSbojXcOvRidhYJ2T
-3YwCpqlC0qC9D2ZmNoukb1a0Pi03MuSJwD/8v9eqwEW9dFAzdnWDzTZMN9CfdjVh
-2qiO5o5Si/X1Dmjdk2F/EM62YJQBAlkZBetFJ0o2QPGTSD+zrpfITIW8Pu+/5zcC
-MZdzyUf0p1GO2Kn7wmqPQjz59zABagmxCNks8HeqPnzmWuADMaggb0nOmrBACE2x
-b9XR6/xaXpwTRf0h5N3evivzUHo6XVw8A3gVUNoBm9Of3PlAsjM4I4SWFb6nrwYv
-RnI04c+R95Su1fMc2wky0PmW+iWRTaEN/cUdX1SF6jo1nRLELGcbMSGUI6UI8kff
-crvCz7uLu7Lr5/CKnEm2bSCZ4eIQpOs4nQ==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem b/pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem
deleted file mode 100644
index eec5c94c0665c..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls-http/user1.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOYqoK+D1FGQFv
-TAxj+XAWkiQC0j1et5ujPyKKL6zFifQhrXEH2U1B8A6KyJV4EOh/yrn5rB1tGDkG
-woS0kwwdeA9I0jLcr+vom0qBNWRJdkmwdZoi3dD+QEvWJs8xFtvm8dMyTkS2Az8y
-dTYd/5EJUbNOCBApItvdVmYlUc19iUaC+5DLTgKCiakH7E9eDovRAMN/VYS3gbnP
-zMeibLrav2HV2pKGbUG32ZGtjDHy14DAyyDw6m1PtPau2rtnavEYxs61p8Zzl6P1
-0LLKyG8MySWAEgbIAALZwMO2z8wl+vnC+j2VUcohGaL9Byl8kcr620OYchDLkt3R
-6qCrdHyjAgMBAAECggEBAK8J8QvytAxJg/T/+7ZC1PTfp1kZNGGDuaV/o2ytuIul
-T//MGQQ+IY8d6Ud9jX9SX84agxalCiP/mkYIbgK0gF7x94yccfTH433ZTxw8yzye
-7SqS41JU7K7mmysaqTkKGSFK0gNlbFMud8f0rxxMJ5dOypMQtZwd63lSkLlwIqcn
-YhKcAdXM4WGv07MFdwAXvrK2trhBBkCA08ZzyVy4lWE+cVfEkwH6O8EdGchl7g4U
-LqIkYzufQWKdH9frt6N3qEV/qs0s1qipVzV07GaPpEdK/G5exJ7NjZaXJkHOuMbt
-7ae1zJBexW41/++fjzsWSYljvSEphjqwrGYrr+tMGNECgYEA+Sg1sHcYO4Bg16qG
-c8XM9g8DFL396X4MnMh+AFrDV8dbdz39x9fFtpKEfqaAZrG6I8fW3RkMLsJvJ4GI
-KDJNz2ezEMbNKlXE9UzvAsrdvWNWDGJvrDu0CbJg2wtBeVEBIUYetSdNHoEwY7ZJ
-QHnwbxYYrUFIJ34rI0SQ17/Z4vsCgYEA1A27jYTWr86JmvjQDUiJcJsbhpa/6OzZ
-n3KTu0n0oCvl7uvvjUfhlzmcq7kyY0oO1C5TQHLiqBaI5hXw+iYwnESLIn7BwhMX
-dcxglvXx95h1NqHYX9GnURl2QtrjmuY5yx+itfmZCRkRPO3c5e9uZyf01CeE4uwl
-hDNh0RrSXHkCgYBvlQhmXQ+nJhk4vI+2LXFbCOISWfvqo562YDu9oOg22Xsm7cZH
-x2QuHXPk3GBInXOFLqwVHHCOSFlLUgFOLykVp5VUABRFz1+Dk86+a2fetywEI9lr
-QtmgNhiWQHY0BIkDA8ogytcIwEaRgUNQ8sswlK68eK39sc1T4BMV7D+CHQKBgG3g
-+8lWBwSsKgOCYBQx/P27caTo4mJosE99yG0o4jhI5ulJmiSEFbINqVAWM7TdQBfU
-NVFU9nuQybknr2l/dnrSzaG/Otk8mVBx6a7vnETm2/3GGV91PJS6c9wqnfu6xkGp
-j99piVH8ikEfI/KFgZi0TJnOLH6FTN9W3J3EnzJJAoGAE4ZLLi+2RP4ZYXI11CJC
-BSM1AEpqemgTUSidZyTiJJMGGrC7tyEba+4TIqeGgvD74p57XFbN7LOJWmnAiK8b
-BLhQqPgOCXqrna00GnNKKx7AzgYHqlq03tGlX858aH18rkSRVk3UhkTkGTSr3iQn
-aJeN/0HbpGr67bimf4bqlCY=
------END PRIVATE KEY-----
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java
index de168284ab47f..af70276aed95e 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java
@@ -55,10 +55,6 @@ public class AuthedAdminProxyHandlerTest extends MockedPulsarServiceBaseTest {
     private BrokerDiscoveryProvider discoveryProvider;
     private PulsarResources resource;
 
-    static String getTlsFile(String name) {
-        return String.format("./src/test/resources/authentication/tls-admin-proxy/%s.pem", name);
-    }
-
     @BeforeMethod
     @Override
     protected void setup() throws Exception {
@@ -67,9 +63,9 @@ protected void setup() throws Exception {
         conf.setAuthorizationEnabled(true);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsTrustCertsFilePath(getTlsFile("ca.cert"));
-        conf.setTlsCertificateFilePath(getTlsFile("broker.cert"));
-        conf.setTlsKeyFilePath(getTlsFile("broker.key-pk8"));
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(false);
         conf.setSuperUserRoles(ImmutableSet.of("admin"));
         conf.setProxyRoles(ImmutableSet.of("proxy"));
@@ -91,15 +87,15 @@ protected void setup() throws Exception {
         proxyConfig.setHttpMaxRequestHeaderSize(20000);
 
         // enable tls and auth&auth at proxy
-        proxyConfig.setTlsCertificateFilePath(getTlsFile("broker.cert"));
-        proxyConfig.setTlsKeyFilePath(getTlsFile("broker.key-pk8"));
-        proxyConfig.setTlsTrustCertsFilePath(getTlsFile("ca.cert"));
+        proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
+        proxyConfig.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
 
         proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
         proxyConfig.setBrokerClientAuthenticationParameters(
                 String.format("tlsCertFile:%s,tlsKeyFile:%s",
-                              getTlsFile("proxy.cert"), getTlsFile("proxy.key-pk8")));
-        proxyConfig.setBrokerClientTrustCertsFilePath(getTlsFile("ca.cert"));
+                              getTlsFileForClient("proxy.cert"), getTlsFileForClient("proxy.key-pk8")));
+        proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         proxyConfig.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName()));
 
         resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper),
@@ -128,22 +124,22 @@ protected void cleanup() throws Exception {
     PulsarAdmin getDirectToBrokerAdminClient(String user) throws Exception {
         return PulsarAdmin.builder()
             .serviceHttpUrl(brokerUrlTls.toString())
-            .tlsTrustCertsFilePath(getTlsFile("ca.cert"))
+            .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
             .allowTlsInsecureConnection(false)
             .authentication(AuthenticationTls.class.getName(),
-                    ImmutableMap.of("tlsCertFile", getTlsFile(user + ".cert"),
-                                    "tlsKeyFile", getTlsFile(user + ".key-pk8")))
+                    ImmutableMap.of("tlsCertFile", getTlsFileForClient(user + ".cert"),
+                                    "tlsKeyFile", getTlsFileForClient(user + ".key-pk8")))
             .build();
     }
 
     PulsarAdmin getAdminClient(String user) throws Exception {
         return PulsarAdmin.builder()
             .serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get())
-            .tlsTrustCertsFilePath(getTlsFile("ca.cert"))
+            .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
             .allowTlsInsecureConnection(false)
             .authentication(AuthenticationTls.class.getName(),
-                    ImmutableMap.of("tlsCertFile", getTlsFile(user + ".cert"),
-                                    "tlsKeyFile", getTlsFile(user + ".key-pk8")))
+                    ImmutableMap.of("tlsCertFile", getTlsFileForClient(user + ".cert"),
+                                    "tlsKeyFile", getTlsFileForClient(user + ".key-pk8")))
             .build();
     }
 
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java
index 5b7cb88f98280..d3291c8fb910d 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java
@@ -51,10 +51,6 @@ public class SuperUserAuthedAdminProxyHandlerTest extends MockedPulsarServiceBas
     private BrokerDiscoveryProvider discoveryProvider;
     private PulsarResources resource;
 
-    static String getTlsFile(String name) {
-        return String.format("./src/test/resources/authentication/tls-admin-proxy/%s.pem", name);
-    }
-
     @BeforeMethod
     @Override
     protected void setup() throws Exception {
@@ -64,9 +60,9 @@ protected void setup() throws Exception {
 
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsTrustCertsFilePath(getTlsFile("ca.cert"));
-        conf.setTlsCertificateFilePath(getTlsFile("broker.cert"));
-        conf.setTlsKeyFilePath(getTlsFile("broker.key-pk8"));
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(false);
         conf.setSuperUserRoles(ImmutableSet.of("admin", "superproxy"));
         conf.setProxyRoles(ImmutableSet.of("superproxy"));
@@ -86,15 +82,15 @@ protected void setup() throws Exception {
         proxyConfig.setTlsEnabledWithBroker(true);
 
         // enable tls and auth&auth at proxy
-        proxyConfig.setTlsCertificateFilePath(getTlsFile("broker.cert"));
-        proxyConfig.setTlsKeyFilePath(getTlsFile("broker.key-pk8"));
-        proxyConfig.setTlsTrustCertsFilePath(getTlsFile("ca.cert"));
+        proxyConfig.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        proxyConfig.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
 
         proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
         proxyConfig.setBrokerClientAuthenticationParameters(
                 String.format("tlsCertFile:%s,tlsKeyFile:%s",
-                              getTlsFile("superproxy.cert"), getTlsFile("superproxy.key-pk8")));
-        proxyConfig.setBrokerClientTrustCertsFilePath(getTlsFile("ca.cert"));
+                              getTlsFileForClient("superproxy.cert"), getTlsFileForClient("superproxy.key-pk8")));
+        proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         proxyConfig.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName()));
 
         resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper),
@@ -123,11 +119,11 @@ protected void cleanup() throws Exception {
     PulsarAdmin getAdminClient(String user) throws Exception {
         return PulsarAdmin.builder()
             .serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get())
-            .tlsTrustCertsFilePath(getTlsFile("ca.cert"))
+            .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
             .allowTlsInsecureConnection(false)
             .authentication(AuthenticationTls.class.getName(),
-                    ImmutableMap.of("tlsCertFile", getTlsFile(user + ".cert"),
-                                    "tlsKeyFile", getTlsFile(user + ".key-pk8")))
+                    ImmutableMap.of("tlsCertFile", getTlsFileForClient(user + ".cert"),
+                                    "tlsKeyFile", getTlsFileForClient(user + ".key-pk8")))
             .build();
     }
 
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem
deleted file mode 100644
index 0665edbdc126c..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEZTCCAk2gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyMjA4NTcwNloYDzIyOTIwNDA2MDg1NzA2WjAQMQ4wDAYDVQQD
-DAVhZG1pbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJw3Jfbn0xkW
-36kqQjES6Hn+YTZ2jXS5Co2MzsGBsIY0qJ2BbWHSSaMrja4IERUaCQp16SWxPmZ0
-srMm6ErDoap+O70CWXLT3ybYMV5aVwv3ca4uxsedzaw9MpFXfUDsJJ3yre1JpO+t
-A/QzJEGq1d6NN49InUP5kB1Rpay3vaxx8hduzqTO+E/Lptv92p6GjOpXi2icSjiA
-pgaan2ldGGKEKv2Sc2bfdIDkTq1yDyNmuPET0yD2dci106EW/mPyj81umPKG/o4K
-5W18yG/IhXw5W1zlgO1fWCuqva8NCBdu7s1c7hUX8DBx7km4/I7dllz/nYHIfCEQ
-Dmj38oQjYk8CAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
-oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
-ZmljYXRlMB0GA1UdDgQWBBQTMzAAOJ9gXvQSS7Be3+qmrb1kVDAfBgNVHSMEGDAW
-gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
-FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQAwy8f6hsG0
-85e3SOIztbUnaVxS7wzDeDzR3vCjpXpm4vTToYzN9zx3JHKSdJrB12emVxItwW/7
-bXqBk0n2EdQjRHCuebnY05eFMNGagMEEMVmSLOproQD7VsyALNxCss1JAyRikh70
-W7wgOVeAhqE53UqqrkzTE7Q+8Bag9t3FytHxApY17XglbWkiVcFpQwSURe9Emi3E
-aCJCryGJXrBNuCFXGzetSygDEy27+2FeH8S2XsMwUEGLqDDehzvMenVz1xjXtq+s
-KPkofAde52NHd4lLkSeBMSFnKe3V7Xxax2OEUsoQRF3bkbpcJSWsKS9ZAA2yrtuy
-Nz/aA1F42LuSFPAYQr1kcZ8eSS918RWz+BiJYU2JuUOPd1XUmJXVvZ4CJurWaC7+
-ZD51YdD8E245xd55fsA6/qLx3eE/Kp0dVq+Hxuz6b4yLET0zkGunOe4A3hnRgkOA
-XolXCL+VthhWtFGXn8CjpxDnzjahq69Io+dINehqd5aJEgvnHZIK2s7FTqqBBodU
-HhyAE94f64z7ziuRhEG54bmBF+MoGyPf6dVn1Mp3+o+YeQ5q6XlKgh+u8jMgmqRO
-ikdsVdMqopt/FXh9eFzvQrwOZFLK6JE/edUgb6xvS1jMF5zi2lIlIkq1RBQOr4HT
-XDwX4vRtfpDxpsetGVpeq9O07fbMvp+mkw==
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem
deleted file mode 100644
index 6aaa22c44d746..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/admin.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCcNyX259MZFt+p
-KkIxEuh5/mE2do10uQqNjM7BgbCGNKidgW1h0kmjK42uCBEVGgkKdeklsT5mdLKz
-JuhKw6Gqfju9Ally098m2DFeWlcL93GuLsbHnc2sPTKRV31A7CSd8q3tSaTvrQP0
-MyRBqtXejTePSJ1D+ZAdUaWst72scfIXbs6kzvhPy6bb/dqehozqV4tonEo4gKYG
-mp9pXRhihCr9knNm33SA5E6tcg8jZrjxE9Mg9nXItdOhFv5j8o/Nbpjyhv6OCuVt
-fMhvyIV8OVtc5YDtX1grqr2vDQgXbu7NXO4VF/Awce5JuPyO3ZZc/52ByHwhEA5o
-9/KEI2JPAgMBAAECggEAP+Ipq2Q4puz8wGBgu1LhMWp+9Nfcl1xI3YQ01VulBe0o
-+2h/g96McKcSBJaV7cw84ENB+kEWpK2amrsRiemhBmkjIvOAAv50Jp2I6u4E5Qbn
-PXUxo1Z8UrCgKmHd/hvUCafByuUwBzf5AvebHyOu3JlhnD302mSHtAW8u/pUHd3D
-sxJaw1zwQvmlD5ryM2IVYSji7NYCXF0H6V7HfyohTCrQFEWEAdqEDcFR1BUwbPCE
-raq7sAiEy+cBUnfV3IOEAffOZy0vSR90/WwERcwrzCdZmpWpTqtbcqtdBPqsSQzX
-shDvXd0e43+FSJzCtQsSQ8WzIrp3rgKJUDA1pJQW4QKBgQDJdTcB6qZz8r+4q9gc
-q1KAJyMy01Vio1yaqYzXr0C9Z5FW1GhL+4fwer2y9JyD45sb/lP4reFj9S193BNR
-C8cdxM5GrWEpzaQ0Dt1s8P1UbU8G6r4NqwI6ORF3CxXZKfXivQcgqBurJGrBdjIC
-NwqAzSkX5flBbhfTlJuUH87k3wKBgQDGgjzdIWXab7FZfdUzzrtEwNooBiSEFixm
-UAwP5sxL8VkM9wzAKPEVDDQBBoKIga9OESif4S9UUo4tu65AYfxF9Om26K4QrVj8
-HT/U+lfT7xFmPd/GINIbeTSmW0w7Ehpj8SbcQI4Sb2lVE562FlHh7QbHZd0/X/2J
-nbgT9MRAkQKBgQCVPAN3o/+SPOzRPFtnQXJoBJYKfIrv+twKpjbzP5vRsvrzO33X
-a4kUF5iXDKU0/lJUtl42BXjFt0Xvyit1CiiCYNv9d0pW0UMmXSyiGxNOi3rTQOlw
-7pFD2Cqb6NZSfMbtI+I3ytBUQzHiBlCdW3CoYVJjpbSzR37W+WsWm0mEOQKBgQCq
-ANObFYUjA1DBMZCrY7rhcL/kUw5myI6RuK/71k7UIwd+oP0cfHOq8N6AmlCkE1xM
-4UkHU1SzRFhbNkZPARuJ1etqJ+8afTqd/3axMQyShkVCaG8CQQ1vVegPKFUqqaBM
-QzRioC6L/zoYEEt16buKXvHVRpmqMszxVE9XV+HS4QKBgDvs5qloOowS5kcWInrj
-yecu5MJvFf2IZpMw7EiKV8VUPeKaiUlqgFj9d9cotUIMauXgBq6f5NBRg7Ike60t
-/JJrPtqXY+gdFLjxcKMUVYhomFlQYYg/RJZUBrtkyKBP68abopCYmb59r3ixeNNf
-qA1F36mmFtzdjSdtH/dTTecN
------END PRIVATE KEY-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem
deleted file mode 100644
index b5c7a5dc709a1..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.cert.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD
-DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
-DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63
-EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O
-OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01
-+ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X
-aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s
-bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw
-ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
-ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA
-DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx
-DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV
-HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ
-ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y
-ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN
-zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9
-spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur
-x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0
-fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d
-mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS
-efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp
-0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C
-cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK
-3N2cGJWRyvpvvmbP2hKCa4eLgEc=
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem
deleted file mode 100644
index 2b51d015b8ace..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/broker.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq
-mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM
-6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU
-TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh
-QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ
-M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN
-95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j
-WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN
-wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y
-3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK
-r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv
-yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP
-eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj
-bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF
-M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP
-PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9
-T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2
-yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP
-UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat
-V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt
-6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm
-3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ
-2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA
-G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+
-I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI
-5qJ7jxgksHWji4lYfL4r4J6Zaw==
------END PRIVATE KEY-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem
deleted file mode 100644
index 0446700135d39..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/ca.cert.pem
+++ /dev/null
@@ -1,29 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFCDCCAvCgAwIBAgIJANfih0+geeIMMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV
-BAMMBmZvb2JhcjAeFw0xODA2MjIwODQ2MjFaFw0zODA2MTcwODQ2MjFaMBExDzAN
-BgNVBAMMBmZvb2JhcjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOVU
-UpTPeXCeyfUiQS824l9s9krZd4R6TA4D97eQ9EWm2D7ppV4gPApHO8j5f+joo/b6
-Iso4aFlHpJ8VV2a5Ol7rjQw43MJHaBgwDxB1XWgsNdfoI7ebtp/BWg2nM3r8wm+Z
-gKenf9d1/1Ol+6yFUehkLkIXUvldiVegmmje8FnwhcDNE1eTrh66XqSJXEXqgBKu
-NqsoYcVak72OyOO1/N8CESoSdyBkbSiH5vJyo0AUCjn7tULga7fxojmqBZDog9Pg
-e5Fi/hbCrdinbxBrMgIxQ7wqXw2sw6iOWu4FU8Ih/CuF4xaQy2YP7MEk4Ff0LCY0
-KMhFMWU7550r/fz/C2l7fKhREyCQPa/bVE+dfxgZ/gCZ+p7vQ154hCCjpd+5bECv
-SN1bcVIPG6ngQu4vMXa7QRBi/Od40jSVGVJXYY6kXvrYatad7035w2GGGGkvMsQm
-y53yh4tqQfH7ulHqB0J5LebTQRp6nRizWigVCLjNkxJYI+Dj51qvT1zdyWEegKr1
-CthBfYzXlfjeH3xri1f0UABeC12n24Wkacd9af7zs7S3rYntEK444w/3fB0F62Lh
-SESfMLAmUH0dF5plRShrFUXz23nUeS8EYgWmnGkpf/HDzB67vdfAK0tfJEtmmY78
-q06OSgMr+AOOqaomh4Ez2ZQG592bS71G8MrE7r2/AgMBAAGjYzBhMB0GA1UdDgQW
-BBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAfBgNVHSMEGDAWgBRXC+nLI+i/Rz5Qej9F
-fqEYQ50VJzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
-9w0BAQsFAAOCAgEAYd2PxdV+YOaWcmMG1fK7CGwSzDOGsgC7hi4gWPiNsVbz6fwQ
-m5Ac7Zw76dzin8gzOPKST7B8WIoc7ZWrMnyh3G6A3u29Ec8iWahqGa91NPA3bOIl
-0ldXnXfa416+JL/Q5utpiV6W2XDaB53v9GqpMk4rOTS9kCFOiuH5ZU8P69jp9mq6
-7pI/+hWFr+21ibmXH6ANxRLd/5+AqojRUYowAu2997Z+xmbpwx/2Svciq3LNY/Vz
-s9DudUHCBHj/DPgNxsEUt8QNohjQkRbFTY0a1aXodJ/pm0Ehk2kf9KwYYYduR7ak
-6UmPIPrZg6FePNahxwMZ0RtgX7EXmpiiIH1q9BsulddWkrFQclevsWO3ONQVrDs2
-gwY0HQuCRCJ+xgS2cyGiGohW5MkIsg1aI0i0j5GIUSppCIYgirAGCairARbCjhcx
-pbMe8RTuBhCqO3R2wZ0wXu7P7/ArI/Ltm1dU6IeHUAUmeneVj5ie0SdA19mHTS2o
-lG77N0jy6eq2zyEwJE6tuS/tyP1xrxdzXCYY7f6X9aNfsuPVQTcnrFajvDv8R6uD
-YnRStVCdS6fZEP0JzsLrqp9bgLIRRsiqsVVBCgJdK1I/X59qk2EyCLXWSgk8T9XZ
-iux8LlPpskt30YYt1KhlWB9zVz7k0uYAwits5foU6RfCRDPAyOa1q/QOXk0=
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem
deleted file mode 100644
index 6c2f4295c9dbd..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEZTCCAk2gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyNTEzNTYwNFoYDzIyOTIwNDA5MTM1NjA0WjAQMQ4wDAYDVQQD
-DAVwcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMieX78Yj8OW
-eBHAneRaCCoO8Qrpj8zGo7h9lCdmi1lBDh1uR2sDbotiHGfJzQn836WcYmyeAvfn
-qvgr9HCXmXdLgmJ3GT/LVu5GEm6msSDiZQPr9so5lQVioisK4UwJROQsE/J52cyR
-9o3H6M4FKb6QpoobKa62fSfTumwwulaYaDJuRRGoGIkcRuUQ59EWAaDkD3IcDpAn
-9mTbnE4Iz+JxSrsZ5DJ3X/m/AqyLWtj6GAfyK9a1dhNdlf2x4JZT1QNtojiBXt95
-OIZyRBNbHMFniq5gel6wdBkmJWutfcTct7wKa2LCxLpKoDIc1HWoL3+RUzOKxYIP
-0qXEQ3bmONkCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
-oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
-ZmljYXRlMB0GA1UdDgQWBBSgsgfmDbXEkrrpHUC9GnDDjxaKizAfBgNVHSMEGDAW
-gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
-FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQB+uZ2OWR+G
-sRqYHEeVqUI8G2p8Np8eC/onGpBL8Gj9SGxIQcIPtqHPUlCe9fd/96JOptOGRYEB
-BmUCaCmQ4IgMW6e6fArka5IB4XXIgHFXyQ6ImTvjDavzlVw06zn9S4dLwVzsRBg+
-GS9svtq23W+f5rEN5N+7LhtcbclfiG4VCCqDG5VhkzEok+SRamDI8rDRZtodMw0O
-/+L+xaaQPUPjX8KUlKn4uVpCDbxUzHonlCPzbkHHm5su0D4ysjJIy3/y3yow6JE/
-02L7PZkmkmw3/V+84T3X8/GD15sVUv/3v1gXEBxYwAs+RNTJ0APvMEMSvCq0AMfF
-bPMZBuAGNBG7lv7TovzHgGFKXT7du5OFF/qjAsEffhbo224CB96fgwvvndwHHBFh
-J06BvHZG1i9dDVhUKoB1owkWrE4RZv2ZKEtZYgizzSmzZRHtARo0t1Byc5djx1tX
-TkJOHshNqJZOY1ER0DPaVQgKI+PRTbEdj/xPGRX3ebSqDmilAfPXshqgElfch6Yl
-f2V58TyCnjXOibvkG9D5OyCdWLEECumOZgYar0KZgNfrvTOi1OKXnX1fsbh29fWA
-ICZRcdmjkz79zQXY2SuzCWlskuXPKAmW1AMqs+l6ormmKfzIUx3Yriy4LqIIYY1v
-uQD5vghZmd9HUg2KaXfSGD9stGCD8KhntA==
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem
deleted file mode 100644
index 70e0107f2741f..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/proxy.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDInl+/GI/DlngR
-wJ3kWggqDvEK6Y/MxqO4fZQnZotZQQ4dbkdrA26LYhxnyc0J/N+lnGJsngL356r4
-K/Rwl5l3S4Jidxk/y1buRhJuprEg4mUD6/bKOZUFYqIrCuFMCUTkLBPyednMkfaN
-x+jOBSm+kKaKGymutn0n07psMLpWmGgybkURqBiJHEblEOfRFgGg5A9yHA6QJ/Zk
-25xOCM/icUq7GeQyd1/5vwKsi1rY+hgH8ivWtXYTXZX9seCWU9UDbaI4gV7feTiG
-ckQTWxzBZ4quYHpesHQZJiVrrX3E3Le8CmtiwsS6SqAyHNR1qC9/kVMzisWCD9Kl
-xEN25jjZAgMBAAECggEAJJyCbKVW1yLGlrbIGbw0cTh41Lz6+SvnBOwl9WrJU2iD
-4usVLXpa2iT1ehthx8jWJ6r6a0gK0qL8mH2tBj8kSpkFGmMRwIqjOqifBIJ3IMEw
-Hh8Z0p3fjDQL1D8QDohCgkFpAn8qOCMLE6S/35khnR1Yxytd1/yFqpcBFm1uFA85
-dYisjPqWm/IZIU5rH0zgKAIhtvl9abnoi93443EHsKpRAW1gwRXx9Aak7TV768bZ
-tELBsaTnXnNzamDiaimmxEOlqR9O0W8JE/31KFL26JcVmsTRG7sMpoUxCEMjOuGZ
-J30bXFZUW6NrDpFsQ7uTqD6TNn2971N8KFCLnC/JYQKBgQDo82axC7L7n7BYTu18
-dupeT7n5dTBD/I3l0KtT05xiZA8GZr2i+pt+/aWzCzK4/Ee4jb4/o8CRQRB5v5mo
-c9lc+BaoAIQiwiw+aufT+UojrcijrOMEL5Zk3zdZ2rcEoAsVvqtejNnwLCGI9Rnl
-gp7n9oRhwDIv9Fu09snUougE3wKBgQDceAGKUB8pGd3eEya/0jU9J60LsKbcJSsN
-4v1S5LiPtHOyhr0g4x/LibMP2PJhG3tJ1bgpaGmn9du2D20M6ukRhIYyn/7G+N+A
-oqryyvO1MMYnhc4IEQvWrzDnBM0hV2bdjp4s/1ASVHVRwk8+orqxysIJ1D75nnRX
-Tyfl6HgBRwKBgQCfVlWIhiMPv6OkU6BXgRNAHTJs8f5okmgQqNF3jgequRwZ2c6e
-muIfU6myNNel9lGsZ6+Y4g4GjMWTMT4OHeewkrUUhv3atIwEyaT2tc5DZ0wUwF2r
-cE1jg9bdbB/BVyMd5YRcMOWlRNpPTq8+8EB3E4RrREZPzMmplyBohGFFawKBgGjc
-P0dM8nU3E1rj2wNTdPUAYQL1Y3fDyeWR+BEsLkhTeNAJ2/y/akkB1oQMGMRtMMee
-ejhfrBkyC+1dCu4g8PffA4EirihvCMcDF7HhK+cbKrRzpNobWXkj3GuU0ggwrQFm
-Kv+V87y0JRTdCZnuBkQ3/vBz3fwWDJnWUVC9sA5TAoGBAN4t2gJCZax5yInUgyWi
-Tgumb2qVWsGBBLMTKIsrkK/KphzgAhHcVhDCybA79TmIM2FfIlpf6TE7Mv4NI055
-ZJzHX+GMT5Czy2Ku9MJD3PLTFN5MjYb6g92fKViLDMI6fwzTHB8xPJ7Ob9bV2srS
-ZlmKNXTkZFk3/orK4WdCZufz
------END PRIVATE KEY-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem
deleted file mode 100644
index 01b69eacc32cb..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.cert.pem
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDCDCCAfCgAwIBAgIJAMPzPm3QygzRMA0GCSqGSIb3DQEBCwUAMBAxDjAMBgNV
-BAMMBXJhbmRvMCAXDTE4MDYyNzA5MDA0OVoYDzIyOTIwNDExMDkwMDQ5WjAQMQ4w
-DAYDVQQDDAVyYW5kbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKR
-nN1y/aBrhbNM6kDfBmWTaKFuANEZ+VST++AVImssN+FyOL2ukxuT74JW0b9g9scl
-aTYzUBbKuTrMIf77RPIJY6R5E6JWn12UhDpnLa9eYGOkyPyoSp5hdyJj3spElWkl
-eGRY8g9zsWJO6QDpTL4g2VAtWTjFIVI3l3enNgNp3hJnJkdvxyBuRN7Szg9hGPmQ
-Tp9rsKkZQD4SJ9/WMAlqSqeB6rSn20rydMJCnI92TT8hlv1wZcVXSVrJypzZb27t
-aAi5o9GEtag3aXalR8zL9DiUk64338A0fYgf2/GSjomZehcg17h8tElYMFV6hJqS
-95gM18nfwldttbihjKMCAwEAAaNjMGEwHQYDVR0OBBYEFAzmiWmqQIB10sSoSAzb
-claQvBPtMB8GA1UdIwQYMBaAFAzmiWmqQIB10sSoSAzbclaQvBPtMA8GA1UdEwEB
-/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IBAQCN7nup
-rGd4sVuP2vvcRbhGo1JCWDS4Uwe7nbSR/4QTAlyUYK8KIwTvfxeEeO4yBGmiDzOE
-+W2y0i22qVnShNCBZ50Evm+FkIEcwlnLtAJt6zIrE4cWdqW1a6CMp9MJkONjACOx
-3Y560SlqQuoKa41PtXUnk8tFT8Asz7b3NtEfmrBlZJpSSM5exBOCH/Enwzac890u
-zJuweTz4aO8aefunkPMKlBPliJ6EkEYclrjJqxi+4VrEMtAIRxwKXPvnHnXESaF7
-yUE4qrjQa4B+Bn1DWqfLcHJCVDxqYrXiEAN/YtX0UOqxZvGd49eNIs2vQkET0+55
-J3hW25OLSnPjAO/j
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem
deleted file mode 100644
index 86a7fa387a8c7..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/randouser.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCykZzdcv2ga4Wz
-TOpA3wZlk2ihbgDRGflUk/vgFSJrLDfhcji9rpMbk++CVtG/YPbHJWk2M1AWyrk6
-zCH++0TyCWOkeROiVp9dlIQ6Zy2vXmBjpMj8qEqeYXciY97KRJVpJXhkWPIPc7Fi
-TukA6Uy+INlQLVk4xSFSN5d3pzYDad4SZyZHb8cgbkTe0s4PYRj5kE6fa7CpGUA+
-Eiff1jAJakqngeq0p9tK8nTCQpyPdk0/IZb9cGXFV0laycqc2W9u7WgIuaPRhLWo
-N2l2pUfMy/Q4lJOuN9/ANH2IH9vxko6JmXoXINe4fLRJWDBVeoSakveYDNfJ38JX
-bbW4oYyjAgMBAAECggEAf8cSqKQQOSq3kYYIWkM9IJJK3LkKfJZJg+wg4Eg/SNFr
-azeAwrqZKbLCQFI/5OJNtFNg5hfxx11pDlnkOcEzpL5zPs4k7pVtlFkiBWivmD3A
-W40fBSynuI2l4kX0tmg9QfA+JhA/pi7zT5WHxc8ryyFWX7kTjzwAjASbrlNIo0d7
-e5SMGcZUeH7m/+pCw626nhpexAJ5Fm9/uc0Zwh4xrzS6j2xoqhOqvQb175V/Iv6/
-8nBq/VjE5LqoP57gqw/Cs4zwwBFayr/WaitGVikga2eZI3q/GFMtez27cdMxEZhI
-9ZP1y8kzlzs1QPjdJQUxIE+/zIm3w8IL4iO0SQE5AQKBgQDmSYazZBkmixUzvBhg
-8tGTWJOAG0NhgBamkbaR6s1L8ou9A5MGbFbRBbc5zRmiyMC2Q4D6Xu0ut2N93tUr
-DkxZa35dt3HWMRmNByjHtpMzapYnmDpuF4k7dy+81GkUbd2ZDYadnknjiaFaDZ7j
-9JxUENIwzzLhFMx7emBQ1+3zewKBgQDGgcewYthAsCTAcBtGKw6G39jAYCpr2N7X
-/B5W5n4VXVHKq35wK1bHRnHj7dkm4sH572XB1tPoMrnQ7j9i+XSkc1Ixo0iHr9H0
-QauVQKQqzkvI5cSaWS5N6ZRk6GOGxI0SwYcNNdqJzEEJuYHCTUpff4d6utMgpTZ6
-SQJw8u0O+QKBgE2VEcNYAr0geDkgslnfFEn+ulqbVL0BSSA+0PIh154xjXBVRvAQ
-CcOLmGnptixIU9xTq50t49wsPmGGc+x4ebJaa40pIznU+tWvRsbZtIfK7eFTAMRc
-O4iEI9oK+Ye/Z7uLegGZ9SyqDmjnU9NaclxD+nwlIfAAcM9csBwsUucHAoGABjJ7
-B3iug6Z8Hz3gvBoQBAns/GSELoXAv0FxuQjNGuGk8gzUj6/qr6H1YEZGpz4hDCp7
-JMgOKYub3XfypqZfC9tFz6LnWsUUaum575jrByMVnpn9v0vVdD08ksHmiYiNVu6P
-xsvNnMuxpBoUgPpkvgJ/Okem27gMsViiKOCMohECgYEAt3MwlVILN0t6RPBCpTKB
-nrh6cxx8kQ2TI1UP44WciBkGVvIQOEbl+1955lt/LAECzooiBJOkKubcPK22sJPS
-MZauEwdAqRQj/uUs3oRISUjo0H/IKGI3O23jhIAI2XC/zkQTE6dKjhXxJIl+5a8f
-v9VhnIlgSfaxu3R0l7SvQlo=
------END PRIVATE KEY-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem
deleted file mode 100644
index 9656e2c8ba0d2..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEajCCAlKgAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyNjEzMjUxN1oYDzIyOTIwNDEwMTMyNTE3WjAVMRMwEQYDVQQD
-DApzdXBlcnByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA43Sn
-ys/h3wxJ/IBEzbJdQAKCxU4of5KgTDzFoOaS8C63Nwbjgy0qcWYdRDceP4lJIKSA
-1+JZ+R1opyrfVIC2D9oDJFIJTFfXy9G9VYDccwAONPgAamvRzBnYcEu8fqM+Ohle
-kZltKktAHnX3WtG3RyxEL5nYzlMlGwUXJu3Rxc9SlkYSxERzjWHikGqGmXLX2qB0
-k6oyxTrK+4+EHk3khbEIqyQZSOFbD7NMfnRy0CWFv/9T1shgjAIBCake6jY7lwaT
-S7JvbLfG6ABf5xHMxoWLXa2qwb+Ar43Ff9g8kKZwFOxMeGcwkzyJPBbWytFxaWn+
-R2RHhTaVCGc22CjdfQIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB
-BAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg
-Q2VydGlmaWNhdGUwHQYDVR0OBBYEFHEgLhfH0Z1QlLxeQbG7YZyBlz1MMB8GA1Ud
-IwQYMBaAFFcL6csj6L9HPlB6P0V+oRhDnRUnMA4GA1UdDwEB/wQEAwIF4DAdBgNV
-HSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBAK9q
-3YnEa99Cq3Vo3g6PE/A/xOE97seVNuavqyBcVv9PrTydb5XG4jPkYU3xOYXIc8rA
-A4gzd+AsGO9rjGMPGGjQJI3JO/3BCeBJMkn50C/rM1yWVMHnVyFcJlg16xaWtj1e
-2Jk8egJW2gSYyF+N2TdzI7tOb002GNr36posnqO+IOoLapyHFBxxUjsPDRoo8fJn
-myWsV1Y9oRUZyJlfIAJsu85ew7gDBY2jaiEiopzour3uU3C0N7gYni2OmVwfr6J8
-R2/Jp43BSD5sYOW9RAJIEEXef+InYtz9HTJvKu2LsWwIBkaztk29tJcDE+1La6Sw
-0dF0YkUwnXoGQFjiV+8pXX3TF5glXKj1rU8WfNazF6lqslB6DmdgR3/FQ6Z2sE86
-d9hVtayZIGlzU0rWmBBtr++7Wo88nmzAtd/xbZMFG8U//+Q2AvJT2oVGtqM48+al
-rnsN/gYrLDr7RC14bHIuO1v6ZL/rAi7SPKrKYAyQVTAcRuW516SxxR6S1Xa1ITnh
-rwgKg13eQuwu3iigguIS9XL6nAXabBIxBxMl6o2YlyIPekKYIcQmpqhkavJ6VOgX
-iq9VdY6fIJVfmxNZuwM3/28k7UeUAfhI2SSVH4ZURbPiGGH2wukc1QkmtZ2cNa35
-C1y79aqJbIa3ErqLFPj/fM+34x8L7QHPq6RfaODa
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem
deleted file mode 100644
index 2e4140b8fb392..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/superproxy.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDjdKfKz+HfDEn8
-gETNsl1AAoLFTih/kqBMPMWg5pLwLrc3BuODLSpxZh1ENx4/iUkgpIDX4ln5HWin
-Kt9UgLYP2gMkUglMV9fL0b1VgNxzAA40+ABqa9HMGdhwS7x+oz46GV6RmW0qS0Ae
-dfda0bdHLEQvmdjOUyUbBRcm7dHFz1KWRhLERHONYeKQaoaZctfaoHSTqjLFOsr7
-j4QeTeSFsQirJBlI4VsPs0x+dHLQJYW//1PWyGCMAgEJqR7qNjuXBpNLsm9st8bo
-AF/nEczGhYtdrarBv4CvjcV/2DyQpnAU7Ex4ZzCTPIk8FtbK0XFpaf5HZEeFNpUI
-ZzbYKN19AgMBAAECggEBAM1JAwuD1drmj3wKFI8F1S2pVndXFCwXnP9RthiDIckO
-kKNkX0CMKgtQ20cu6+jyMgL5FaRCkWvJxCNkCU6OIENsQ3urYuL5QTWeZeBevhg4
-y5m43z8tcptgFD09zbEKCmaLcRO9wo3yfrs/QvE/58efxyajFs8YsZuSW5Px/msl
-AFN8qGY0q0lQbQPK8cPYT4OnW9isqEjkvHq1Q+ryv4cxsGWtBZYmJVeLVHzClihi
-lODdN6YsDcu9jLwsA8o2WSBRsbLHK6caW4FdEwgOLckc0EGGEU64VEB9ZOwgcwYQ
-M9mZDs2w7qk63YnDH4KmpoP0Oc8N+bG7Pkifd7f8HqECgYEA8esMQya6Qln4O7Qo
-aI8X9t5gCpq+bh/P+7vYdBR/9St5VtE/P0yuaQdT2e5t+Ei4ujqHvJ0GTPmqYIJf
-2DNvJMu8EnXv+nqKQCsyMc/nqQN1CcRGqZssH1V/ZIQLRUA0cN4hzE0eHITwp6U9
-vD/WaE3mKX/XHthIjc8hnuoajOkCgYEA8LIYMgyD3wClWOKe5TkBumI0KvA9tP90
-IFHu1wLNl0tKBpsXkzzxiav1FMX3K7B29vxukrs946KRESHzRg4c5ULPnMmwcQyx
-AortKJWrGVsna/QDWPRutXSN1XmnjKWsHmTpQQqLfaRvLW+Hd34+EVdVh9sVt/y9
-RucnBvxcX3UCgYBebm/U7pMaP2BkfcigN+sU1G0M9qaK+iQHkaXGehIQs62js/5K
-STZzjQawNR/8IPbqytodR/YjqflVvs6G6FzkMhrx4dORJLA+qB3pz8wP72eKLnGe
-1xF8EbWumNSFbbCKtkrfIuM0IriF2Dym9QxOnsnPPTXNtoNrx4TKMXu3sQKBgBq9
-noSI8WmoF7adTsvmnnOHj4YptKFUNCGXGLLYg+DII4xCVMct4SPLb+oD6Gb5Lu5X
-sy0oEkMk/3roy68/yCQMXSZtHeYhY9UFfD2jCyRBBUswC+MpHNeaAFv0LRIqIcoq
-qeNo+YBW8WcZ2fIDm3+vtTfntiz/rkOfUK2tAdI1AoGAL2LVDF5e1meSRocEoy8e
-gqrshrhg+KBqYmcjtd4Iup4WvR6uyH4qE9yFLLHFZ04pZzXLHcPfqgOSP8qTxgH3
-h0uqtcYmejS/yl6PbC9OPXcSkMgntRkTbU9Ug05ijfN9NRnW+8WXvi8Z6DKed1P1
-9PxwA75P9o0h00mMk8PQitQ=
------END PRIVATE KEY-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem
deleted file mode 100644
index 072f2867fe62b..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.cert.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEZTCCAk2gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyNzA4NDAyNVoYDzIyOTIwNDExMDg0MDI1WjAQMQ4wDAYDVQQD
-DAV1c2VyMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM5iqgr4PUUZ
-AW9MDGP5cBaSJALSPV63m6M/IoovrMWJ9CGtcQfZTUHwDorIlXgQ6H/KufmsHW0Y
-OQbChLSTDB14D0jSMtyv6+ibSoE1ZEl2SbB1miLd0P5AS9YmzzEW2+bx0zJORLYD
-PzJ1Nh3/kQlRs04IECki291WZiVRzX2JRoL7kMtOAoKJqQfsT14Oi9EAw39VhLeB
-uc/Mx6Jsutq/YdXakoZtQbfZka2MMfLXgMDLIPDqbU+09q7au2dq8RjGzrWnxnOX
-o/XQssrIbwzJJYASBsgAAtnAw7bPzCX6+cL6PZVRyiEZov0HKXyRyvrbQ5hyEMuS
-3dHqoKt0fKMCAwEAAaOBxTCBwjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
-oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
-ZmljYXRlMB0GA1UdDgQWBBQ7NSD6lx6Vq38cEoD5l7FHs1Ej8DAfBgNVHSMEGDAW
-gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
-FAYIKwYBBQUHAwIGCCsGAQUFBwMEMA0GCSqGSIb3DQEBCwUAA4ICAQC+cU7ctY7o
-eTW+oHTq9EGJUMwW1fOww4QDrHtgZT4OYkO88zxQV2Cr050p8eaV5dHXZBf9/bRV
-7hPNV5+HpQhb9TZ9xK2WRZ2QV7a/UDyUnksVKGSK9tNZMZPueOEB19e4bIBcgnQa
-5i9sgZr93na7pFOY7lBQy6gfaOcnejYHmvVqIGaZBVH8rkEsGhhkJxy7qkpFNKgf
-PGiIRo9L0WYqDCSiaICeCiteJwIfjsUFJKF0YnpXZq1kFfQscnleg60MWZAXvacp
-tAciE51Ow60cqQWER66iwqnBSPD4l91SxAaGQAmalgCioGsYSbojXcOvRidhYJ2T
-3YwCpqlC0qC9D2ZmNoukb1a0Pi03MuSJwD/8v9eqwEW9dFAzdnWDzTZMN9CfdjVh
-2qiO5o5Si/X1Dmjdk2F/EM62YJQBAlkZBetFJ0o2QPGTSD+zrpfITIW8Pu+/5zcC
-MZdzyUf0p1GO2Kn7wmqPQjz59zABagmxCNks8HeqPnzmWuADMaggb0nOmrBACE2x
-b9XR6/xaXpwTRf0h5N3evivzUHo6XVw8A3gVUNoBm9Of3PlAsjM4I4SWFb6nrwYv
-RnI04c+R95Su1fMc2wky0PmW+iWRTaEN/cUdX1SF6jo1nRLELGcbMSGUI6UI8kff
-crvCz7uLu7Lr5/CKnEm2bSCZ4eIQpOs4nQ==
------END CERTIFICATE-----
diff --git a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem b/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem
deleted file mode 100644
index eec5c94c0665c..0000000000000
--- a/pulsar-proxy/src/test/resources/authentication/tls-admin-proxy/user1.key-pk8.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOYqoK+D1FGQFv
-TAxj+XAWkiQC0j1et5ujPyKKL6zFifQhrXEH2U1B8A6KyJV4EOh/yrn5rB1tGDkG
-woS0kwwdeA9I0jLcr+vom0qBNWRJdkmwdZoi3dD+QEvWJs8xFtvm8dMyTkS2Az8y
-dTYd/5EJUbNOCBApItvdVmYlUc19iUaC+5DLTgKCiakH7E9eDovRAMN/VYS3gbnP
-zMeibLrav2HV2pKGbUG32ZGtjDHy14DAyyDw6m1PtPau2rtnavEYxs61p8Zzl6P1
-0LLKyG8MySWAEgbIAALZwMO2z8wl+vnC+j2VUcohGaL9Byl8kcr620OYchDLkt3R
-6qCrdHyjAgMBAAECggEBAK8J8QvytAxJg/T/+7ZC1PTfp1kZNGGDuaV/o2ytuIul
-T//MGQQ+IY8d6Ud9jX9SX84agxalCiP/mkYIbgK0gF7x94yccfTH433ZTxw8yzye
-7SqS41JU7K7mmysaqTkKGSFK0gNlbFMud8f0rxxMJ5dOypMQtZwd63lSkLlwIqcn
-YhKcAdXM4WGv07MFdwAXvrK2trhBBkCA08ZzyVy4lWE+cVfEkwH6O8EdGchl7g4U
-LqIkYzufQWKdH9frt6N3qEV/qs0s1qipVzV07GaPpEdK/G5exJ7NjZaXJkHOuMbt
-7ae1zJBexW41/++fjzsWSYljvSEphjqwrGYrr+tMGNECgYEA+Sg1sHcYO4Bg16qG
-c8XM9g8DFL396X4MnMh+AFrDV8dbdz39x9fFtpKEfqaAZrG6I8fW3RkMLsJvJ4GI
-KDJNz2ezEMbNKlXE9UzvAsrdvWNWDGJvrDu0CbJg2wtBeVEBIUYetSdNHoEwY7ZJ
-QHnwbxYYrUFIJ34rI0SQ17/Z4vsCgYEA1A27jYTWr86JmvjQDUiJcJsbhpa/6OzZ
-n3KTu0n0oCvl7uvvjUfhlzmcq7kyY0oO1C5TQHLiqBaI5hXw+iYwnESLIn7BwhMX
-dcxglvXx95h1NqHYX9GnURl2QtrjmuY5yx+itfmZCRkRPO3c5e9uZyf01CeE4uwl
-hDNh0RrSXHkCgYBvlQhmXQ+nJhk4vI+2LXFbCOISWfvqo562YDu9oOg22Xsm7cZH
-x2QuHXPk3GBInXOFLqwVHHCOSFlLUgFOLykVp5VUABRFz1+Dk86+a2fetywEI9lr
-QtmgNhiWQHY0BIkDA8ogytcIwEaRgUNQ8sswlK68eK39sc1T4BMV7D+CHQKBgG3g
-+8lWBwSsKgOCYBQx/P27caTo4mJosE99yG0o4jhI5ulJmiSEFbINqVAWM7TdQBfU
-NVFU9nuQybknr2l/dnrSzaG/Otk8mVBx6a7vnETm2/3GGV91PJS6c9wqnfu6xkGp
-j99piVH8ikEfI/KFgZi0TJnOLH6FTN9W3J3EnzJJAoGAE4ZLLi+2RP4ZYXI11CJC
-BSM1AEpqemgTUSidZyTiJJMGGrC7tyEba+4TIqeGgvD74p57XFbN7LOJWmnAiK8b
-BLhQqPgOCXqrna00GnNKKx7AzgYHqlq03tGlX858aH18rkSRVk3UhkTkGTSr3iQn
-aJeN/0HbpGr67bimf4bqlCY=
------END PRIVATE KEY-----

From b07f2a9b85a0f22b89fed5b51ee8625b00b7262e Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 11 May 2023 06:48:59 +0300
Subject: [PATCH 268/404] [fix][test] Fix flaky test
 ConnectionPoolTest.testSetProxyToTargetBrokerAddress (#20293)

---
 .../client/impl/ConnectionPoolTest.java       | 35 ++++++++++---------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java
index fb564bd5083c1..fe0aa4dd4953b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java
@@ -154,7 +154,7 @@ public void testEnableConnectionPool() throws Exception {
     @Test
     public void testSetProxyToTargetBrokerAddress() throws Exception {
         ClientConfigurationData conf = new ClientConfigurationData();
-        conf.setConnectionsPerBroker(5);
+        conf.setConnectionsPerBroker(1);
 
 
         EventLoopGroup eventLoop =
@@ -176,19 +176,24 @@ protected void doResolve(SocketAddress socketAddress, Promise promise) throws Ex
             @Override
             protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws Exception {
                 final InetSocketAddress socketAddress1 = (InetSocketAddress) socketAddress;
-                final boolean isProxy = socketAddress1.getHostName().equals("proxy");
-                final boolean isBroker = socketAddress1.getHostName().equals("broker");
+                String hostName = socketAddress1.getHostName();
+                final boolean isProxy = hostName.equals("proxy");
+                final boolean isBroker = hostName.startsWith("broker");
                 if (!isProxy && !isBroker) {
                     promise.setFailure(new IllegalStateException());
                     throw new IllegalStateException();
                 }
                 List result = new ArrayList<>();
+                // All 127.0.0.0/8 addresses are valid local loopback addresses
                 if (isProxy) {
-                    result.add(new InetSocketAddress("localhost", brokerPort));
-                    result.add(InetSocketAddress.createUnresolved("proxy", brokerPort));
-                } else {
-                    result.add(new InetSocketAddress("127.0.0.1", brokerPort));
-                    result.add(InetSocketAddress.createUnresolved("broker", brokerPort));
+                    result.add(new InetSocketAddress("127.0.0.101", brokerPort));
+                    result.add(new InetSocketAddress("127.0.0.102", brokerPort));
+                } else if (hostName.equals("broker1")){
+                    result.add(new InetSocketAddress("127.0.0.103", brokerPort));
+                    result.add(new InetSocketAddress("127.0.0.104", brokerPort));
+                } else if (hostName.equals("broker2")){
+                    result.add(new InetSocketAddress("127.0.0.105", brokerPort));
+                    result.add(new InetSocketAddress("127.0.0.106", brokerPort));
                 }
                 promise.setSuccess(result);
             }
@@ -203,23 +208,19 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws
                 InetSocketAddress.createUnresolved("proxy", 9999)).get();
         Assert.assertEquals(cnx.remoteHostName, "proxy");
         Assert.assertNull(cnx.proxyToTargetBrokerAddress);
-        cnx.close();
 
         cnx = pool.getConnection(
-                InetSocketAddress.createUnresolved("broker", 9999),
+                InetSocketAddress.createUnresolved("broker1", 9999),
                 InetSocketAddress.createUnresolved("proxy", 9999)).get();
         Assert.assertEquals(cnx.remoteHostName, "proxy");
-        Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker:9999");
-        cnx.close();
+        Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker1:9999");
 
 
         cnx = pool.getConnection(
-                InetSocketAddress.createUnresolved("broker", 9999),
-                InetSocketAddress.createUnresolved("broker", 9999)).get();
-        Assert.assertEquals(cnx.remoteHostName, "broker");
+                InetSocketAddress.createUnresolved("broker2", 9999),
+                InetSocketAddress.createUnresolved("broker2", 9999)).get();
+        Assert.assertEquals(cnx.remoteHostName, "broker2");
         Assert.assertNull(cnx.proxyToTargetBrokerAddress);
-        cnx.close();
-
 
         pool.closeAllConnections();
         pool.close();

From 4b24c9e85ad534c2087063e7873eded4f9ab7a21 Mon Sep 17 00:00:00 2001
From: Shen Liu 
Date: Thu, 11 May 2023 22:28:40 +0800
Subject: [PATCH 269/404] [fix][broker] Fix NPE cause by topic publish rate
 limiter. (#20302)

Co-authored-by: druidliu 
---
 .../java/org/apache/pulsar/broker/service/AbstractTopic.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 6245ce19eebc6..8269dc0c3d16f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -115,7 +115,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener
Date: Thu, 11 May 2023 17:29:22 -0500
Subject: [PATCH 270/404] [fix][fn] Correct TLS cert config translation from
 broker to fn worker (#20297)

### Motivation

The `initializeWorkerConfigFromBrokerConfig` method converts a `ServiceConfiguration` object to a `WorkerConfig`. This is used when the function worker is running within the broker and when pulsar is running in standalone mode. The TLS certificates that are trusted by the broker should also be trusted by the function worker, and the TLS certificates that are trusted by the broker client should be trusted by the function worker's client.

### Modifications

* Improve the `PulsarFunctionTlsTest` test by adding awaitility. Before this change, the test was flaky on my machine.
* Fix the mapping of broker config to function worker config.

### Verifying this change

A test is added. Note that the old test doesn't technically fail due to the misconfiguration at the moment. The error is in the logs. Here is one of the stack traces that is gone after this change:

```
2023-05-10T23:19:27,087 - WARN  - [pulsar-client-io-253-3:ClientCnx@344] - [localhost/127.0.0.1:59715] Got exception io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:499)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.handshakeException(ReferenceCountedOpenSslEngine.java:1907)
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.wrap(ReferenceCountedOpenSslEngine.java:834)
	at java.base/javax.net.ssl.SSLEngine.wrap(SSLEngine.java:564)
	at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:1042)
	at io.netty.handler.ssl.SslHandler.wrapNonAppData(SslHandler.java:928)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1418)
	at io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1256)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1296)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
	... 17 more
Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
	at java.base/sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:439)
	at java.base/sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:306)
	at java.base/sun.security.validator.Validator.validate(Validator.java:264)
	at java.base/sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285)
	at java.base/sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144)
	at io.netty.handler.ssl.ReferenceCountedOpenSslClientContext$ExtendedTrustManagerVerifyCallback.verify(ReferenceCountedOpenSslClientContext.java:234)
	at io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:779)
	at io.netty.internal.tcnative.CertificateVerifierTask.runTask(CertificateVerifierTask.java:36)
	at io.netty.internal.tcnative.SSLTask.run(SSLTask.java:48)
	at io.netty.internal.tcnative.SSLTask.run(SSLTask.java:42)
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.runAndResetNeedTask(ReferenceCountedOpenSslEngine.java:1496)
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.access$700(ReferenceCountedOpenSslEngine.java:94)
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine$TaskDecorator.run(ReferenceCountedOpenSslEngine.java:1471)
	at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1558)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1404)
	... 21 more
Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
	at java.base/sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141)
	at java.base/sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126)
	at java.base/java.security.cert.CertPathBuilder.build(CertPathBuilder.java:297)
	at java.base/sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:434)
	... 35 more
```

### Does this pull request potentially affect one of the following parts:

This affects the configuration, but I think it should be classified as fixing a bug, so it is not a breaking change.

### Documentation

- [x] `doc-not-needed`
### Matching PR in forked repository

PR in forked repository: the modified test passes locally, so skipping forked test.
---
 .../apache/pulsar/broker/PulsarService.java   |  3 ++-
 .../worker/PulsarFunctionTlsTest.java         | 23 +++++++++++++++----
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 5fc9920d0f215..f833674ceeb0d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -1859,7 +1859,8 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu
         workerConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection());
         workerConfig.setTlsEnabled(brokerConfig.isTlsEnabled());
         workerConfig.setTlsEnableHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled());
-        workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath());
+        workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getBrokerClientTrustCertsFilePath());
+        workerConfig.setTlsTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath());
 
         // client in worker will use this config to authenticate with broker
         workerConfig.setBrokerClientAuthenticationPlugin(brokerConfig.getBrokerClientAuthenticationPlugin());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
index 00db6a65b16ea..246d980d6178c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
@@ -51,6 +51,7 @@
 import org.apache.pulsar.functions.sink.PulsarSink;
 import org.apache.pulsar.functions.worker.service.WorkerServiceLoader;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -65,6 +66,7 @@ public class PulsarFunctionTlsTest {
     private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
     private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
     private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
+    private static final String CA_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
 
     LocalBookkeeperEnsemble bkEnsemble;
     protected PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT];
@@ -111,9 +113,9 @@ void setup() throws Exception {
             config.setAuthenticationProviders(providers);
             config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
             config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-            config.setTlsAllowInsecureConnection(true);
+            config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
             config.setBrokerClientTlsEnabled(true);
-            config.setBrokerClientTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
+            config.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
             config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
             config.setBrokerClientAuthenticationParameters(
                 "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + ",tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH);
@@ -141,7 +143,6 @@ void setup() throws Exception {
             workerConfig.setUseTls(true);
             workerConfig.setTlsEnableHostnameVerification(true);
             workerConfig.setTlsAllowInsecureConnection(false);
-            workerConfig.setBrokerClientTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
             fnWorkerServices[i] = WorkerServiceLoader.load(workerConfig);
 
             configurations[i] = config;
@@ -163,8 +164,7 @@ void setup() throws Exception {
 
             pulsarAdmins[i] = PulsarAdmin.builder()
                 .serviceHttpUrl(pulsarServices[i].getWebServiceAddressTls())
-                .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH)
-                .allowTlsInsecureConnection(true)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .authentication(authTls)
                 .build();
         }
@@ -216,6 +216,13 @@ void tearDown() throws Exception {
         }
     }
 
+    @Test
+    public void testTLSTrustCertsConfigMapping() throws Exception {
+        WorkerConfig workerConfig = fnWorkerServices[0].getWorkerConfig();
+        assertEquals(workerConfig.getTlsTrustCertsFilePath(), CA_CERT_FILE_PATH);
+        assertEquals(workerConfig.getBrokerClientTrustCertsFilePath(), CA_CERT_FILE_PATH);
+    }
+
     @Test
     public void testFunctionsCreation() throws Exception {
 
@@ -233,6 +240,12 @@ public void testFunctionsCreation() throws Exception {
                 functionConfig, jarFilePathUrl
             );
 
+            // Function creation is not strongly consistent, so this test can fail with a get that is too eager and
+            // does not have retries.
+            final PulsarAdmin admin = pulsarAdmins[i];
+            Awaitility.await().ignoreExceptions()
+                    .untilAsserted(() -> admin.functions().getFunction(testTenant, "my-ns", functionName));
+
             FunctionConfig config = pulsarAdmins[i].functions().getFunction(testTenant, "my-ns", functionName);
             assertEquals(config.getTenant(), testTenant);
             assertEquals(config.getNamespace(), "my-ns");

From 94c7bf3e57f6fc4323b36142fba651c4ad21d301 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Fri, 12 May 2023 12:27:49 +0800
Subject: [PATCH 271/404] [fix][build] Fix publish image script (#20305)

Signed-off-by: Zixuan Liu 
---
 docker/publish.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker/publish.sh b/docker/publish.sh
index af0d72d4b3437..45b338d85f8ef 100755
--- a/docker/publish.sh
+++ b/docker/publish.sh
@@ -62,11 +62,11 @@ set -x
 # Fail if any of the subsequent commands fail
 set -e
 
-docker tag pulsar:latest ${docker_registry_org}/pulsar:latest
-docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:latest
+docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:latest
+docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:latest
 
-docker tag pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION
-docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION
+docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION
+docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION
 
 # Push all images and tags
 docker push ${docker_registry_org}/pulsar:latest

From 5d5ec947249df50bf35a78b6a2a0d3b00d97ca66 Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Fri, 12 May 2023 19:29:42 +0800
Subject: [PATCH 272/404] [fix][broker] Allow Access to System Topic Metadata
 for Reader Creation Post-Namespace Deletion (#20304)

## Motivation
After initiating the snapshot segment function, deletion of topics necessitates the activation of readers. Furthermore, these readers should be opened and deleted as they are used, which implies that we should not pre-store readers. However, after initiating the deletion of namespaces currently, it is not allowed to obtain the metadata of partition topics or lookup, making it impossible to create readers. This results in the inability to delete namespaces.

## Modification
Allow the acquisition of system topic metadata after initiating namespace deletion, thus creating readers to clean up topic data.
---
 .../admin/impl/PersistentTopicsBase.java      | 14 +++++-
 .../pulsar/broker/lookup/TopicLookupBase.java | 44 +++++++++++--------
 .../broker/transaction/TransactionTest.java   | 21 +++++++++
 3 files changed, 58 insertions(+), 21 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 9dcbe13d6159b..ed7cd70c6411c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.admin.impl;
 
+import static org.apache.pulsar.common.naming.SystemTopicNames.isSystemTopic;
 import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign;
 import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName;
 import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX;
@@ -4408,8 +4409,13 @@ public CompletableFuture getPartitionedTopicMetadata(
         // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can
         // serve/redirect request else fail partitioned-metadata-request so, client fails while creating
         // producer/consumer
+        // It is necessary for system topic operations because system topics are used to store metadata
+        // and other vital information. Even after namespace starting deletion,,
+        // we need to access the metadata of system topics to create readers and clean up topic data.
+        // If we don't do this, it can prevent namespace deletion due to inaccessible readers.
         authorizationFuture.thenCompose(__ ->
-                        checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject()))
+                        checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(),
+                                SystemTopicNames.isSystemTopic(topicName)))
                 .thenCompose(res ->
                         pulsar.getBrokerService().fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName))
                 .thenAccept(metadata -> {
@@ -4436,7 +4442,11 @@ public static CompletableFuture unsafeGetPartitionedTo
         // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can
         // serve/redirect request else fail partitioned-metadata-request so, client fails while creating
         // producer/consumer
-        checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject())
+        // It is necessary for system topic operations because system topics are used to store metadata
+        // and other vital information. Even after namespace starting deletion,,
+        // we need to access the metadata of system topics to create readers and clean up topic data.
+        // If we don't do this, it can prevent namespace deletion due to inaccessible readers.
+        checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(), isSystemTopic(topicName))
             .thenCompose(res -> pulsar.getBrokerService()
                 .fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName))
             .thenAccept(metadata -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java
index 3b64d2a9f8393..bd70201cba55d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java
@@ -41,6 +41,7 @@
 import org.apache.pulsar.common.api.proto.ServerError;
 import org.apache.pulsar.common.lookup.data.LookupData;
 import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.NamespaceOperation;
@@ -221,26 +222,31 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe
                 // (2) authorize client
                 checkAuthorizationAsync(pulsarService, topicName, clientAppId, authenticationData).thenRun(() -> {
                         // (3) validate global namespace
+                        // It is necessary for system topic operations because system topics are used to store metadata
+                        // and other vital information. Even after namespace starting deletion,
+                        // we need to access the metadata of system topics to create readers and clean up topic data.
+                        // If we don't do this, it can prevent namespace deletion due to inaccessible readers.
                         checkLocalOrGetPeerReplicationCluster(pulsarService,
-                                topicName.getNamespaceObject()).thenAccept(peerClusterData -> {
-                            if (peerClusterData == null) {
-                                // (4) all validation passed: initiate lookup
-                                validationFuture.complete(null);
-                                return;
-                            }
-                            // if peer-cluster-data is present it means namespace is owned by that peer-cluster and
-                            // request should be redirect to the peer-cluster
-                            if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl())
-                                    && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) {
-                                validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError,
-                                        "Redirected cluster's brokerService url is not configured",
-                                        requestId));
-                                return;
-                            }
-                            validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(),
-                                    peerClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect,
-                                    requestId,
-                                    false));
+                                topicName.getNamespaceObject(), SystemTopicNames.isSystemTopic(topicName))
+                                .thenAccept(peerClusterData -> {
+                                    if (peerClusterData == null) {
+                                        // (4) all validation passed: initiate lookup
+                                        validationFuture.complete(null);
+                                        return;
+                                    }
+                                    // if peer-cluster-data is present it means namespace is owned by that peer-cluster
+                                    // and request should be redirect to the peer-cluster
+                                    if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl())
+                                            && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) {
+                                        validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError,
+                                                "Redirected cluster's brokerService url is not configured",
+                                                requestId));
+                                        return;
+                                    }
+                                    validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(),
+                                            peerClusterData.getBrokerServiceUrlTls(), true,
+                                            LookupType.Redirect, requestId,
+                                            false));
                         }).exceptionally(ex -> {
                             Throwable throwable = FutureUtil.unwrapCompletionException(ex);
                             if (throwable instanceof RestException restException){
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
index c3533e70cf8be..c4ec2ec766e32 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
@@ -272,6 +272,27 @@ public void testCreateTransactionSystemTopic() throws Exception {
         }
     }
 
+    @Test
+    public void testCanDeleteNamespaceWhenEnableTxnSegmentedSnapshot() throws Exception {
+        // Enable the segmented snapshot feature
+        pulsarServiceList.get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
+        pulsarServiceList.get(0).getConfig().setForceDeleteNamespaceAllowed(true);
+
+        // Create a new namespace
+        String namespaceName =  TENANT + "/testSegmentedSnapshotWithoutCreatingOldSnapshotTopic";
+        admin.namespaces().createNamespace(namespaceName);
+
+        // Create a new topic in the namespace
+        String topicName = "persistent://" + namespaceName + "/newTopic";
+        @Cleanup
+        Producer producer = pulsarClient.newProducer().topic(topicName).create();
+        producer.close();
+
+        // Destroy the namespace after the test
+        admin.namespaces().deleteNamespace(namespaceName, true);
+        pulsarServiceList.get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(false);
+    }
+
     @Test
     public void brokerNotInitTxnManagedLedgerTopic() throws Exception {
         String subName = "test";

From 7e54e9b0463f5c69846dc44892d88281d5508466 Mon Sep 17 00:00:00 2001
From: "Kim, Joo Hyuk" 
Date: Mon, 15 May 2023 16:29:50 +0900
Subject: [PATCH 273/404] [fix][broker] Fix class name typo
 `PrecisPublishLimiter` to "Precise" (#20310)

---
 .../pulsar/broker/service/AbstractTopic.java  |  2 +-
 ...imiter.java => PrecisePublishLimiter.java} | 10 +++---
 .../PrecisTopicPublishRateThrottleTest.java   |  2 +-
 ...st.java => PrecisePublishLimiterTest.java} | 30 ++++++++--------
 .../service/PublishRateLimiterTest.java       | 34 ++++++++++---------
 5 files changed, 40 insertions(+), 38 deletions(-)
 rename pulsar-broker/src/main/java/org/apache/pulsar/broker/service/{PrecisPublishLimiter.java => PrecisePublishLimiter.java} (93%)
 rename pulsar-broker/src/test/java/org/apache/pulsar/broker/service/{PrecisPublishLimiterTest.java => PrecisePublishLimiterTest.java} (57%)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 8269dc0c3d16f..4614b846c8eee 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -1274,7 +1274,7 @@ public void updatePublishDispatcher() {
                     || this.topicPublishRateLimiter == PublishRateLimiter.DISABLED_RATE_LIMITER) {
                     // create new rateLimiter if rate-limiter is disabled
                     if (preciseTopicPublishRateLimitingEnable) {
-                        this.topicPublishRateLimiter = new PrecisPublishLimiter(publishRate,
+                        this.topicPublishRateLimiter = new PrecisePublishLimiter(publishRate,
                             () -> this.enableCnxAutoRead(), brokerService.pulsar().getExecutor());
                     } else {
                         this.topicPublishRateLimiter = new PublishRateLimiterImpl(publishRate);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java
similarity index 93%
rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java
rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java
index 6e215b3b49515..ce14f6d7dd71e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java
@@ -24,7 +24,7 @@
 import org.apache.pulsar.common.util.RateLimitFunction;
 import org.apache.pulsar.common.util.RateLimiter;
 
-public class PrecisPublishLimiter implements PublishRateLimiter {
+public class PrecisePublishLimiter implements PublishRateLimiter {
     protected volatile int publishMaxMessageRate = 0;
     protected volatile long publishMaxByteRate = 0;
     // precise mode for publish rate limiter
@@ -33,18 +33,18 @@ public class PrecisPublishLimiter implements PublishRateLimiter {
     private final RateLimitFunction rateLimitFunction;
     private final ScheduledExecutorService scheduledExecutorService;
 
-    public PrecisPublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) {
+    public PrecisePublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) {
         this.rateLimitFunction = rateLimitFunction;
         update(policies, clusterName);
         this.scheduledExecutorService = null;
     }
 
-    public PrecisPublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) {
+    public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) {
         this(publishRate, rateLimitFunction, null);
     }
 
-    public PrecisPublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction,
-                                ScheduledExecutorService scheduledExecutorService) {
+    public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction,
+                                 ScheduledExecutorService scheduledExecutorService) {
         this.rateLimitFunction = rateLimitFunction;
         update(publishRate);
         this.scheduledExecutorService = scheduledExecutorService;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java
index 07632814378d0..c22ed41fc1533 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java
@@ -164,7 +164,7 @@ public void testBrokerLevelPublishRateDynamicUpdate() throws Exception{
                     "" + rateInMsg));
         Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get();
         Assert.assertNotNull(topicRef);
-        PrecisPublishLimiter limiter = ((PrecisPublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter);
+        PrecisePublishLimiter limiter = ((PrecisePublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter);
         Awaitility.await().untilAsserted(() -> Assert.assertEquals(limiter.publishMaxMessageRate, rateInMsg));
         Assert.assertEquals(limiter.publishMaxByteRate, 0);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java
similarity index 57%
rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java
rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java
index a0d0df52d2417..73cb43d52b112 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java
@@ -23,35 +23,35 @@
 import org.apache.pulsar.common.policies.data.PublishRate;
 import org.testng.annotations.Test;
 
-public class PrecisPublishLimiterTest {
+public class PrecisePublishLimiterTest {
 
     @Test
     void shouldResetMsgLimitAfterUpdate() {
-        PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(), () -> {
+        PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> {
         });
-        precisPublishLimiter.update(new PublishRate(1, 1));
-        assertFalse(precisPublishLimiter.tryAcquire(99, 99));
-        precisPublishLimiter.update(new PublishRate(-1, 100));
-        assertTrue(precisPublishLimiter.tryAcquire(99, 99));
+        precisePublishLimiter.update(new PublishRate(1, 1));
+        assertFalse(precisePublishLimiter.tryAcquire(99, 99));
+        precisePublishLimiter.update(new PublishRate(-1, 100));
+        assertTrue(precisePublishLimiter.tryAcquire(99, 99));
     }
 
     @Test
     void shouldResetBytesLimitAfterUpdate() {
-        PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(), () -> {
+        PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> {
         });
-        precisPublishLimiter.update(new PublishRate(1, 1));
-        assertFalse(precisPublishLimiter.tryAcquire(99, 99));
-        precisPublishLimiter.update(new PublishRate(100, -1));
-        assertTrue(precisPublishLimiter.tryAcquire(99, 99));
+        precisePublishLimiter.update(new PublishRate(1, 1));
+        assertFalse(precisePublishLimiter.tryAcquire(99, 99));
+        precisePublishLimiter.update(new PublishRate(100, -1));
+        assertTrue(precisePublishLimiter.tryAcquire(99, 99));
     }
 
     @Test
     void shouldCloseResources() throws Exception {
         for (int i = 0; i < 20000; i++) {
-            PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(100, 100), () -> {
+            PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(100, 100), () -> {
             });
-            precisPublishLimiter.tryAcquire(99, 99);
-            precisPublishLimiter.close();
+            precisePublishLimiter.tryAcquire(99, 99);
+            precisePublishLimiter.close();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java
index 9d2bd49ba04bd..b934ced08c5db 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java
@@ -39,7 +39,7 @@ public class PublishRateLimiterTest {
     private final PublishRate publishRate = new PublishRate(10, 100);
     private final PublishRate newPublishRate = new PublishRate(20, 200);
 
-    private PrecisPublishLimiter precisPublishLimiter;
+    private PrecisePublishLimiter precisePublishLimiter;
     private PublishRateLimiterImpl publishRateLimiter;
 
     @BeforeMethod
@@ -47,7 +47,7 @@ public void setup() throws Exception {
         policies.publishMaxMessageRate = new HashMap<>();
         policies.publishMaxMessageRate.put(CLUSTER_NAME, publishRate);
 
-        precisPublishLimiter = new PrecisPublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit"));
+        precisePublishLimiter = new PrecisePublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit"));
         publishRateLimiter = new PublishRateLimiterImpl(policies, CLUSTER_NAME);
     }
 
@@ -88,23 +88,25 @@ public void testPublishRateLimiterImplUpdate() {
 
     @Test
     public void testPrecisePublishRateLimiterUpdate() {
-        assertFalse(precisPublishLimiter.tryAcquire(15, 150));
+        assertFalse(precisePublishLimiter.tryAcquire(15, 150));
 
         //update
-        precisPublishLimiter.update(newPublishRate);
-        assertTrue(precisPublishLimiter.tryAcquire(15, 150));
+        precisePublishLimiter.update(newPublishRate);
+        assertTrue(precisePublishLimiter.tryAcquire(15, 150));
     }
 
     @Test
     public void testPrecisePublishRateLimiterAcquire() throws Exception {
-        Class precisPublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisPublishLimiter");
-        Field topicPublishRateLimiterOnMessageField = precisPublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage");
-        Field topicPublishRateLimiterOnByteField = precisPublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte");
+        Class precisePublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisePublishLimiter");
+        Field topicPublishRateLimiterOnMessageField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage");
+        Field topicPublishRateLimiterOnByteField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte");
         topicPublishRateLimiterOnMessageField.setAccessible(true);
         topicPublishRateLimiterOnByteField.setAccessible(true);
 
-        RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get(precisPublishLimiter);
-        RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get(precisPublishLimiter);
+        RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get(
+                precisePublishLimiter);
+        RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get(
+                precisePublishLimiter);
 
         Method renewTopicPublishRateLimiterOnMessageMethod = topicPublishRateLimiterOnMessage.getClass().getDeclaredMethod("renew", null);
         Method renewTopicPublishRateLimiterOnByteMethod = topicPublishRateLimiterOnByte.getClass().getDeclaredMethod("renew", null);
@@ -112,7 +114,7 @@ public void testPrecisePublishRateLimiterAcquire() throws Exception {
         renewTopicPublishRateLimiterOnByteMethod.setAccessible(true);
 
         // running tryAcquire in order to lazyInit the renewTask
-        precisPublishLimiter.tryAcquire(1, 10);
+        precisePublishLimiter.tryAcquire(1, 10);
 
         Field onMessageRenewTaskField = topicPublishRateLimiterOnMessage.getClass().getDeclaredField("renewTask");
         Field onByteRenewTaskField = topicPublishRateLimiterOnByte.getClass().getDeclaredField("renewTask");
@@ -129,30 +131,30 @@ public void testPrecisePublishRateLimiterAcquire() throws Exception {
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
 
         // tryAcquire not exceeded
-        assertTrue(precisPublishLimiter.tryAcquire(1, 10));
+        assertTrue(precisePublishLimiter.tryAcquire(1, 10));
         renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage);
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
 
         // tryAcquire numOfMessages exceeded
-        assertFalse(precisPublishLimiter.tryAcquire(11, 100));
+        assertFalse(precisePublishLimiter.tryAcquire(11, 100));
         renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage);
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
 
         // tryAcquire msgSizeInBytes exceeded
-        assertFalse(precisPublishLimiter.tryAcquire(10, 101));
+        assertFalse(precisePublishLimiter.tryAcquire(10, 101));
         renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage);
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
         renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage);
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
 
         // tryAcquire exceeded exactly
-        assertFalse(precisPublishLimiter.tryAcquire(10, 100));
+        assertFalse(precisePublishLimiter.tryAcquire(10, 100));
         renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage);
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
         renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage);
         renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte);
 
         // tryAcquire not exceeded
-        assertTrue(precisPublishLimiter.tryAcquire(9, 99));
+        assertTrue(precisePublishLimiter.tryAcquire(9, 99));
     }
 }

From c5ceec72d6d9362c680c08c3ecb5b1f3e9cc682a Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Mon, 15 May 2023 16:09:12 +0800
Subject: [PATCH 274/404] [fix][test] Flaky-test:
 SimpleProducerConsumerTestStreamingDispatcherTest. rest (#20286)

---
 .../org/apache/pulsar/broker/service/BrokerServiceTest.java     | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index 24e38438c5329..90b00bacaaa88 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -172,8 +172,6 @@ public void testShutDownWithMaxConcurrentUnload() throws Exception {
             assertTrue(e instanceof PulsarClientException.TimeoutException);
         }
 
-        pulsar = null;
-
         producer.close();
         resetState();
     }

From 21afdfd6df57ed56438a3434cd081d9b6dd4132d Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 15 May 2023 20:34:51 +0800
Subject: [PATCH 275/404] [improve][misc] Enable rebase and merge to handle one
 PR of multiple semantic commits (#20325)

---
 .asf.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.asf.yaml b/.asf.yaml
index 459dcb6e2fe57..5c3b8cb98d964 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -35,12 +35,9 @@ github:
     # Enable projects for project management boards
     projects: true
   enabled_merge_buttons:
-    # enable squash button:
     squash:  true
-    # disable merge button:
     merge:   false
-    # disable rebase button:
-    rebase:  false
+    rebase:  true
   protected_branches:
     master:
       required_status_checks:

From 7b54664c364a7360f30cd37f0aa989ab13a14af4 Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 15 May 2023 10:47:00 +0800
Subject: [PATCH 276/404] Revert "[fix][client] Seek should be thread-safe
 (#20242)"

This reverts commit bc1764f9ef71dd31e8cd61c7571e493442bc6395.
---
 .../pulsar/client/impl/ConsumerImpl.java      | 109 ++++++++----------
 .../pulsar/client/impl/ConsumerImplTest.java  |  27 +----
 2 files changed, 50 insertions(+), 86 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 2693f12d3ea8e..8a06ec122b5e6 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -208,7 +208,6 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle
 
     private final AtomicReference clientCnxUsedForConsumerRegistration = new AtomicReference<>();
     private final List previousExceptions = new CopyOnWriteArrayList();
-
     static  ConsumerImpl newConsumerImpl(PulsarClientImpl client,
                                                String topic,
                                                ConsumerConfigurationData conf,
@@ -252,12 +251,10 @@ static  ConsumerImpl newConsumerImpl(PulsarClientImpl client,
     }
 
     protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf,
-                           ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer,
-                           boolean parentConsumerHasListener, CompletableFuture> subscribeFuture,
-                           MessageId startMessageId,
-                           long startMessageRollbackDurationInSec, Schema schema,
-                           ConsumerInterceptors interceptors,
-                           boolean createTopicIfDoesNotExist) {
+           ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer,
+           boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId,
+           long startMessageRollbackDurationInSec, Schema schema, ConsumerInterceptors interceptors,
+           boolean createTopicIfDoesNotExist) {
         super(client, topic, conf, conf.getReceiverQueueSize(), executorProvider, subscribeFuture, schema,
                 interceptors);
         this.consumerId = client.newConsumerId();
@@ -329,21 +326,21 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat
         }
 
         this.connectionHandler = new ConnectionHandler(this,
-                new BackoffBuilder()
-                        .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(),
-                                TimeUnit.NANOSECONDS)
-                        .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS)
-                        .setMandatoryStop(0, TimeUnit.MILLISECONDS)
-                        .create(),
+                        new BackoffBuilder()
+                                .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(),
+                                        TimeUnit.NANOSECONDS)
+                                .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS)
+                                .setMandatoryStop(0, TimeUnit.MILLISECONDS)
+                                .create(),
                 this);
 
         this.topicName = TopicName.get(topic);
         if (this.topicName.isPersistent()) {
             this.acknowledgmentsGroupingTracker =
-                    new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup());
+                new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup());
         } else {
             this.acknowledgmentsGroupingTracker =
-                    NonPersistentAcknowledgmentGroupingTracker.of();
+                NonPersistentAcknowledgmentGroupingTracker.of();
         }
 
         if (conf.getDeadLetterPolicy() != null) {
@@ -420,16 +417,16 @@ public CompletableFuture unsubscribeAsync() {
                 log.error("[{}][{}] Failed to unsubscribe: {}", topic, subscription, e.getCause().getMessage());
                 setState(State.Ready);
                 unsubscribeFuture.completeExceptionally(
-                        PulsarClientException.wrap(e.getCause(),
-                                String.format("Failed to unsubscribe the subscription %s of topic %s",
-                                        topicName.toString(), subscription)));
+                    PulsarClientException.wrap(e.getCause(),
+                        String.format("Failed to unsubscribe the subscription %s of topic %s",
+                            topicName.toString(), subscription)));
                 return null;
             });
         } else {
             unsubscribeFuture.completeExceptionally(
-                    new PulsarClientException(
-                            String.format("The client is not connected to the broker when unsubscribing the "
-                                    + "subscription %s of the topic %s", subscription, topicName.toString())));
+                new PulsarClientException(
+                    String.format("The client is not connected to the broker when unsubscribing the "
+                            + "subscription %s of the topic %s", subscription, topicName.toString())));
         }
         return unsubscribeFuture;
     }
@@ -1410,7 +1407,7 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien
     }
 
     private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId,
-                                        MessageIdData messageId, ClientCnx cnx) {
+            MessageIdData messageId, ClientCnx cnx) {
 
         // Lazy task scheduling to expire incomplete chunk message
         if (expireTimeOfIncompleteChunkedMessageMillis > 0 && expireChunkMessageTaskScheduled.compareAndSet(false,
@@ -1455,7 +1452,7 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m
             increaseAvailablePermits(cnx);
             if (expireTimeOfIncompleteChunkedMessageMillis > 0
                     && System.currentTimeMillis() > (msgMetadata.getPublishTime()
-                    + expireTimeOfIncompleteChunkedMessageMillis)) {
+                            + expireTimeOfIncompleteChunkedMessageMillis)) {
                 doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null);
             } else {
                 trackMessage(msgId);
@@ -1646,7 +1643,7 @@ protected void trackMessage(Message msg) {
     }
 
     protected void trackMessage(MessageId messageId) {
-        trackMessage(messageId, 0);
+            trackMessage(messageId, 0);
     }
 
     protected void trackMessage(MessageId messageId, int redeliveryCount) {
@@ -1787,7 +1784,7 @@ private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, int redeliveryCo
     }
 
     private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload,
-                                              ClientCnx currentCnx, boolean checkMaxMessageSize) {
+            ClientCnx currentCnx, boolean checkMaxMessageSize) {
         CompressionType compressionType = msgMetadata.getCompression();
         CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
         int uncompressedSize = msgMetadata.getUncompressedSize();
@@ -1840,7 +1837,7 @@ private void discardCorruptedMessage(MessageIdImpl messageId, ClientCnx currentC
     }
 
     private void discardCorruptedMessage(MessageIdData messageId, ClientCnx currentCnx,
-                                         ValidationError validationError) {
+            ValidationError validationError) {
         log.error("[{}][{}] Discarding corrupted message at {}:{}", topic, subscription, messageId.getLedgerId(),
                 messageId.getEntryId());
         discardMessage(messageId, currentCnx, validationError);
@@ -2032,8 +2029,8 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId)
                     String originTopicNameStr = getOriginTopicNameStr(message);
                     TypedMessageBuilder typedMessageBuilderNew =
                             producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get()))
-                                    .value(message.getData())
-                                    .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr));
+                            .value(message.getData())
+                            .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr));
                     if (message.hasKey()) {
                         typedMessageBuilderNew.key(message.getKey());
                     }
@@ -2062,7 +2059,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId)
                                 }
                                 result.complete(false);
                                 return null;
-                            });
+                    });
                 }
             }, internalPinnedExecutor).exceptionally(ex -> {
                 log.error("Dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), ex);
@@ -2161,15 +2158,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek,
         final CompletableFuture seekFuture = new CompletableFuture<>();
         ClientCnx cnx = cnx();
 
-        if (!duringSeek.compareAndSet(false, true)) {
-            log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}",
-                    topic, subscription, seekBy);
-            seekFuture.cancel(true);
-            return seekFuture;
-        }
-
         MessageIdAdv originSeekMessageId = seekMessageId;
         seekMessageId = (MessageIdAdv) seekId;
+        duringSeek.set(true);
         log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy);
 
         cnx.sendRequestWithId(seek, requestId).thenRun(() -> {
@@ -2187,9 +2178,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek,
             log.error("[{}][{}] Failed to reset subscription: {}", topic, subscription, e.getCause().getMessage());
 
             seekFuture.completeExceptionally(
-                    PulsarClientException.wrap(e.getCause(),
-                            String.format("Failed to seek the subscription %s of the topic %s to %s",
-                                    subscription, topicName.toString(), seekBy)));
+                PulsarClientException.wrap(e.getCause(),
+                    String.format("Failed to seek the subscription %s of the topic %s to %s",
+                        subscription, topicName.toString(), seekBy)));
             return null;
         });
         return seekFuture;
@@ -2201,7 +2192,7 @@ public CompletableFuture seekAsync(long timestamp) {
         return seekAsyncCheckState(seekBy).orElseGet(() -> {
             long requestId = client.newRequestId();
             return seekAsyncInternal(requestId, Commands.newSeek(consumerId, requestId, timestamp),
-                    MessageId.earliest, seekBy);
+                MessageId.earliest, seekBy);
         });
     }
 
@@ -2367,11 +2358,10 @@ public CompletableFuture> getLastMessageIdsAsync() {
     public CompletableFuture internalGetLastMessageIdAsync() {
         if (getState() == State.Closing || getState() == State.Closed) {
             return FutureUtil
-                    .failedFuture(new PulsarClientException.AlreadyClosedException(
-                            String.format("The consumer %s was already closed when the subscription %s of the topic %s "
-                                            + "getting the last message id", consumerName, subscription,
-                                    topicName.toString())));
-        }
+                .failedFuture(new PulsarClientException.AlreadyClosedException(
+                    String.format("The consumer %s was already closed when the subscription %s of the topic %s "
+                            + "getting the last message id", consumerName, subscription, topicName.toString())));
+                }
 
         AtomicLong opTimeoutMs = new AtomicLong(client.getConfiguration().getOperationTimeoutMs());
         Backoff backoff = new BackoffBuilder()
@@ -2393,12 +2383,11 @@ private void internalGetLastMessageIdAsync(final Backoff backoff,
         if (isConnected() && cnx != null) {
             if (!Commands.peerSupportsGetLastMessageId(cnx.getRemoteEndpointProtocolVersion())) {
                 future.completeExceptionally(
-                        new PulsarClientException.NotSupportedException(
-                                String.format(
-                                        "The command `GetLastMessageId` is not supported for the protocol version %d. "
-                                                + "The consumer is %s, topic %s, subscription %s",
-                                        cnx.getRemoteEndpointProtocolVersion(),
-                                        consumerName, topicName.toString(), subscription)));
+                    new PulsarClientException.NotSupportedException(
+                        String.format("The command `GetLastMessageId` is not supported for the protocol version %d. "
+                                        + "The consumer is %s, topic %s, subscription %s",
+                                cnx.getRemoteEndpointProtocolVersion(),
+                                consumerName, topicName.toString(), subscription)));
                 return;
             }
 
@@ -2417,31 +2406,31 @@ private void internalGetLastMessageIdAsync(final Backoff backoff,
                 }
                 if (log.isDebugEnabled()) {
                     log.debug("[{}][{}] Successfully getLastMessageId {}:{}",
-                            topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId());
+                        topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId());
                 }
 
                 MessageId lastMsgId = lastMessageId.getBatchIndex() <= 0
                         ? new MessageIdImpl(lastMessageId.getLedgerId(),
-                        lastMessageId.getEntryId(), lastMessageId.getPartition())
+                                lastMessageId.getEntryId(), lastMessageId.getPartition())
                         : new BatchMessageIdImpl(lastMessageId.getLedgerId(), lastMessageId.getEntryId(),
-                        lastMessageId.getPartition(), lastMessageId.getBatchIndex());
+                                lastMessageId.getPartition(), lastMessageId.getBatchIndex());
 
                 future.complete(new GetLastMessageIdResponse(lastMsgId, markDeletePosition));
             }).exceptionally(e -> {
                 log.error("[{}][{}] Failed getLastMessageId command", topic, subscription);
                 future.completeExceptionally(
-                        PulsarClientException.wrap(e.getCause(),
-                                String.format("The subscription %s of the topic %s gets the last message id was failed",
-                                        subscription, topicName.toString())));
+                    PulsarClientException.wrap(e.getCause(),
+                        String.format("The subscription %s of the topic %s gets the last message id was failed",
+                            subscription, topicName.toString())));
                 return null;
             });
         } else {
             long nextDelay = Math.min(backoff.next(), remainingTime.get());
             if (nextDelay <= 0) {
                 future.completeExceptionally(
-                        new PulsarClientException.TimeoutException(
-                                String.format("The subscription %s of the topic %s could not get the last message id "
-                                        + "withing configured timeout", subscription, topicName.toString())));
+                    new PulsarClientException.TimeoutException(
+                        String.format("The subscription %s of the topic %s could not get the last message id "
+                                + "withing configured timeout", subscription, topicName.toString())));
                 return;
             }
 
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
index ab11675f5434c..29d180f5f9a16 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.client.impl;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
@@ -27,9 +26,7 @@
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertTrue;
-import io.netty.buffer.ByteBuf;
+
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
@@ -262,26 +259,4 @@ public void testTopicPriorityLevel() {
 
         assertThat(consumer.getPriorityLevel()).isEqualTo(1);
     }
-
-    @Test(invocationTimeOut = 1000)
-    public void testSeekAsyncInternal() {
-        // given
-        ClientCnx cnx = mock(ClientCnx.class);
-        CompletableFuture clientReq = new CompletableFuture<>();
-        when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq);
-
-        consumer.setClientCnx(cnx);
-        consumer.setState(HandlerState.State.Ready);
-
-        // when
-        CompletableFuture firstResult = consumer.seekAsync(1L);
-        CompletableFuture secondResult = consumer.seekAsync(1L);
-
-        clientReq.complete(null);
-
-        // then
-        assertTrue(firstResult.isDone());
-        assertTrue(secondResult.isCancelled());
-        verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong());
-    }
 }

From 0acf8f89d6057170e990b128936175fc5dd33be3 Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 15 May 2023 15:10:11 +0800
Subject: [PATCH 277/404] [fix][client] thread-safe seek

Signed-off-by: tison 
---
 .../pulsar/client/impl/ConsumerImpl.java      | 11 +++++++-
 .../pulsar/client/impl/ConsumerImplTest.java  | 27 ++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 8a06ec122b5e6..4a84e765065f2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -2158,9 +2158,18 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek,
         final CompletableFuture seekFuture = new CompletableFuture<>();
         ClientCnx cnx = cnx();
 
+        if (!duringSeek.compareAndSet(false, true)) {
+            final String message = String.format(
+                    "[%s][%s] attempting to seek operation that is already in progress (seek by %s)",
+                    topic, subscription, seekBy);
+            log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}",
+                    topic, subscription, seekBy);
+            seekFuture.completeExceptionally(new IllegalStateException(message));
+            return seekFuture;
+        }
+
         MessageIdAdv originSeekMessageId = seekMessageId;
         seekMessageId = (MessageIdAdv) seekId;
-        duringSeek.set(true);
         log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy);
 
         cnx.sendRequestWithId(seek, requestId).thenRun(() -> {
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
index 29d180f5f9a16..5a223d5da15c0 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.client.impl;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
@@ -26,7 +27,9 @@
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertTrue;
+import io.netty.buffer.ByteBuf;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
@@ -259,4 +262,26 @@ public void testTopicPriorityLevel() {
 
         assertThat(consumer.getPriorityLevel()).isEqualTo(1);
     }
+
+    @Test(invocationTimeOut = 1000)
+    public void testSeekAsyncInternal() {
+        // given
+        ClientCnx cnx = mock(ClientCnx.class);
+        CompletableFuture clientReq = new CompletableFuture<>();
+        when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq);
+
+        consumer.setClientCnx(cnx);
+        consumer.setState(HandlerState.State.Ready);
+
+        // when
+        CompletableFuture firstResult = consumer.seekAsync(1L);
+        CompletableFuture secondResult = consumer.seekAsync(1L);
+
+        clientReq.complete(null);
+
+        // then
+        assertTrue(firstResult.isDone());
+        assertTrue(secondResult.isCompletedExceptionally());
+        verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong());
+    }
 }

From 8b929e6bf1b1718431abebf04420d7817ad8cdb7 Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Tue, 16 May 2023 09:41:35 +0800
Subject: [PATCH 278/404] [fix][txn] Implement compatibility for transaction
 buffer segmented snapshot feature upgrade (#20235)

master https://github.com/apache/pulsar/issues/16913
## Motivation:
The transaction buffer segmented snapshot feature aims to improve the transaction buffer's performance by segmenting the snapshot and managing it more efficiently. However, for existing topics that were created before this feature was introduced, we need to ensure a seamless transition and compatibility when enabling the segmented snapshot feature.

## Modifications:
1. Updated the `recoverFromSnapshot()` method to read from another topic if the `persistentSnapshotIndexes` is null. This ensures that the appropriate snapshot data is fetched during the recovery process when upgrading to the segmented snapshot feature.
2. Created a new test `testSnapshotProcessorUpdate()` that verifies the compatibility of the transaction

### Verifying this change

- [ ] Make sure that the change passes the CI checks.

*(Please pick either of the following options)*

This change is a trivial rework / code cleanup without any test coverage.

*(or)*

This change is already covered by existing tests, such as *(please describe tests)*.

*(or)*

This change added tests and can be verified as follows:

*(example:)*
  - *Added integration tests for end-to-end deployment with large payloads (10MB)*
  - *Extended integration test for recovery after broker failure*
---
 ...napshotSegmentAbortedTxnProcessorImpl.java |  66 ++++++++-
 .../SegmentAbortedTxnProcessorTest.java       | 133 +++++++++++++++++-
 2 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
index 751c03aff95a9..4f4e58ac3f55b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
@@ -47,6 +47,7 @@
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata;
@@ -265,7 +266,7 @@ public CompletableFuture recoverFromSnapshot() {
                     PositionImpl finalStartReadCursorPosition = startReadCursorPosition;
                     TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes;
                     if (persistentSnapshotIndexes == null) {
-                        return CompletableFuture.completedFuture(null);
+                        return recoverOldSnapshot();
                     } else {
                         this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes
                                 .getSnapshot().getAborts());
@@ -378,6 +379,69 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob
                         .getExecutor(this));
     }
 
+    // This method will be deprecated and removed in version 4.x.0
+    private CompletableFuture recoverOldSnapshot() {
+        return topic.getBrokerService().getTopic(TopicName.get(topic.getName()).getNamespace() + "/"
+                        + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, false)
+                .thenCompose(topicOption -> {
+                    if (!topicOption.isPresent()) {
+                        return CompletableFuture.completedFuture(null);
+                    } else {
+                        return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory()
+                                .getTxnBufferSnapshotService()
+                                .createReader(TopicName.get(topic.getName())).thenComposeAsync(snapshotReader -> {
+                                    PositionImpl startReadCursorPositionInOldSnapshot = null;
+                                    try {
+                                        while (snapshotReader.hasMoreEvents()) {
+                                            Message message = snapshotReader.readNextAsync()
+                                                    .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS);
+                                            if (topic.getName().equals(message.getKey())) {
+                                                TransactionBufferSnapshot transactionBufferSnapshot =
+                                                        message.getValue();
+                                                if (transactionBufferSnapshot != null) {
+                                                    handleOldSnapshot(transactionBufferSnapshot);
+                                                    startReadCursorPositionInOldSnapshot = PositionImpl.get(
+                                                            transactionBufferSnapshot.getMaxReadPositionLedgerId(),
+                                                            transactionBufferSnapshot.getMaxReadPositionEntryId());
+                                                }
+                                            }
+                                        }
+                                    } catch (TimeoutException ex) {
+                                        Throwable t = FutureUtil.unwrapCompletionException(ex);
+                                        String errorMessage = String.format("[%s] Transaction buffer recover fail by "
+                                                + "read transactionBufferSnapshot timeout!", topic.getName());
+                                        log.error(errorMessage, t);
+                                        return FutureUtil.failedFuture(new BrokerServiceException
+                                                .ServiceUnitNotReadyException(errorMessage, t));
+                                    } catch (Exception ex) {
+                                        log.error("[{}] Transaction buffer recover fail when read "
+                                                + "transactionBufferSnapshot!", topic.getName(), ex);
+                                        return FutureUtil.failedFuture(ex);
+                                    } finally {
+                                        assert snapshotReader != null;
+                                        closeReader(snapshotReader);
+                                    }
+                                    return CompletableFuture.completedFuture(startReadCursorPositionInOldSnapshot);
+                                },
+                                        topic.getBrokerService().getPulsar().getTransactionExecutorProvider()
+                                                .getExecutor(this));
+                    }
+                });
+    }
+
+    // This method will be deprecated and removed in version 4.x.0
+    private void handleOldSnapshot(TransactionBufferSnapshot snapshot) {
+        if (snapshot.getAborts() != null) {
+            snapshot.getAborts().forEach(abortTxnMetadata -> {
+                TxnID txnID = new TxnID(abortTxnMetadata.getTxnIdMostBits(),
+                        abortTxnMetadata.getTxnIdLeastBits());
+                aborts.put(txnID, txnID);
+                //The old data will be written into the first segment.
+                unsealedTxnIds.add(txnID);
+            });
+        }
+    }
+
     @Override
     public CompletableFuture clearAbortedTxnSnapshot() {
         return persistentWorker.appendTask(PersistentWorker.OperationType.Clear,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index c157d7cf8c527..cb15ab003f7b7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.transaction;
 
+import static org.junit.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
@@ -44,10 +45,16 @@
 import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.transaction.Transaction;
 import org.apache.pulsar.client.api.transaction.TxnID;
 import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
@@ -247,8 +254,8 @@ public void testClearSnapshotSegments() throws Exception {
     private void verifySnapshotSegmentsSize(String topic, int size) throws Exception {
         SystemTopicClient.Reader reader =
                 pulsarService.getTransactionBufferSnapshotServiceFactory()
-                .getTxnBufferSnapshotSegmentService()
-                .createReader(TopicName.get(topic)).get();
+                        .getTxnBufferSnapshotSegmentService()
+                        .createReader(TopicName.get(topic)).get();
         int segmentCount = 0;
         while (reader.hasMoreEvents()) {
             Message message = reader.readNextAsync()
@@ -286,4 +293,126 @@ private void doCompaction(TopicName topic) throws Exception {
         CompletableFuture compactionFuture = (CompletableFuture) field.get(snapshotTopic);
         org.awaitility.Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone()));
     }
+
+    /**
+     * This test verifies the compatibility of the transaction buffer segmented snapshot feature
+     * when enabled on an existing topic.
+     * It performs the following steps:
+     * 1. Creates a topic with segmented snapshot disabled.
+     * 2. Sends 10 messages without using transactions.
+     * 3. Sends 10 messages using transactions and aborts them.
+     * 4. Sends 10 messages without using transactions.
+     * 5. Verifies that only the non-transactional messages are received.
+     * 6. Enables the segmented snapshot feature and sets the snapshot segment size.
+     * 7. Unloads the topic.
+     * 8. Sends a new message using a transaction and aborts it.
+     * 9. Verifies that the topic has exactly one segment.
+     * 10. Re-subscribes the consumer and re-verifies that only the non-transactional messages are received.
+     */
+    @Test
+    public void testSnapshotProcessorUpgrade() throws Exception {
+        this.pulsarService = getPulsarServiceList().get(0);
+        this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false);
+
+        // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions.
+        // Abort these transactions and verify the data.
+        final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade";
+        Producer producer = pulsarClient.newProducer().topic(topicName).create();
+        Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe();
+
+        // Send 10 messages without using transactions
+        for (int i = 0; i < 10; i++) {
+            producer.send(("test-message-" + i).getBytes());
+        }
+
+        // Send 10 messages using transactions and abort them
+        for (int i = 0; i < 10; i++) {
+            Transaction txn = pulsarClient.newTransaction()
+                    .withTransactionTimeout(5, TimeUnit.SECONDS)
+                    .build().get();
+            producer.newMessage(txn).value(("test-txn-message-" + i).getBytes()).sendAsync();
+            txn.abort().get();
+        }
+
+        // Send 10 messages without using transactions
+        for (int i = 10; i < 20; i++) {
+            producer.send(("test-message-" + i).getBytes());
+        }
+
+        // Verify the data
+        for (int i = 0; i < 20; i++) {
+            Message msg = consumer.receive(5, TimeUnit.SECONDS);
+            assertEquals("test-message-" + i, new String(msg.getData()));
+        }
+
+        // Enable segmented snapshot
+        this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
+        this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() +
+                SEGMENT_SIZE * 3);
+
+        // Unload the topic
+        admin.topics().unload(topicName);
+
+        // Sends a new message using a transaction and aborts it.
+        Transaction txn = pulsarClient.newTransaction()
+                .withTransactionTimeout(5, TimeUnit.SECONDS)
+                .build().get();
+        producer.newMessage(txn).value("test-message-new".getBytes()).send();
+        txn.abort().get();
+
+        // Verifies that the topic has exactly one segment.
+        Awaitility.await().untilAsserted(() -> {
+            String segmentTopic = "persistent://" + NAMESPACE1 + "/" +
+                    SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS;
+            TopicStats topicStats = admin.topics().getStats(segmentTopic);
+            assertEquals(1, topicStats.getMsgInCounter());
+        });
+
+        // Re-subscribes the consumer and re-verifies that only the non-transactional messages are received.
+        consumer.close();
+        consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe();
+        for (int i = 0; i < 20; i++) {
+            Message msg = consumer.receive(5, TimeUnit.SECONDS);
+            assertEquals("test-message-" + i, new String(msg.getData()));
+        }
+    }
+
+    /**
+     * This test verifies that when the segmented snapshot feature is enabled, creating a new topic
+     * does not create a __transaction_buffer_snapshot topic in the same namespace.
+     * The test performs the following steps:
+     * 1. Enable the segmented snapshot feature.
+     * 2. Create a new namespace.
+     * 3. Create a new topic in the namespace.
+     * 4. Check that the __transaction_buffer_snapshot topic is not created in the same namespace.
+     * 5. Destroy the namespace after the test.
+     */
+    @Test
+    public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Exception {
+        // Enable the segmented snapshot feature
+        pulsarService = getPulsarServiceList().get(0);
+        pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
+
+        // Create a new namespace
+        String namespaceName = "tnx/testSegmentedSnapshotWithoutCreatingOldSnapshotTopic";
+        admin.namespaces().createNamespace(namespaceName);
+
+        // Create a new topic in the namespace
+        String topicName = "persistent://" + namespaceName + "/newTopic";
+        Producer producer = pulsarClient.newProducer().topic(topicName).create();
+        producer.close();
+
+        // Check that the __transaction_buffer_snapshot topic is not created in the same namespace
+        String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" +
+                SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT;
+        try {
+            admin.topics().getStats(transactionBufferSnapshotTopic);
+            fail("The __transaction_buffer_snapshot topic should not exist");
+        } catch (PulsarAdminException e) {
+            assertEquals(e.getStatusCode(), 404);
+        }
+
+        // Destroy the namespace after the test
+        admin.namespaces().deleteNamespace(namespaceName, true);
+    }
 }

From 426ad3e53ba621c442bce89eb208add8f9d1563e Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Tue, 16 May 2023 17:19:46 +0800
Subject: [PATCH 279/404] [fix][ml] Fix ledger left in OPEN state when enable
 `inactiveLedgerRollOverTimeMs` (#20276)

close `currentLegder` after roll current ledger if full
---
 .../mledger/impl/ManagedLedgerImpl.java       | 31 ++++++++++++++++---
 .../mledger/impl/ManagedLedgerTest.java       | 15 +++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index c74db884a95a9..15e9d332fa103 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1775,15 +1775,19 @@ public void closeComplete(int rc, LedgerHandle lh, Object o) {
                                   + "acked ledgerId %s", currentLedger.getId(), lh.getId());
 
                     if (rc == BKException.Code.OK) {
-                        log.debug("Successfully closed ledger {}", lh.getId());
+                        if (log.isDebugEnabled()) {
+                            log.debug("[{}] Successfully closed ledger {}, trigger by rollover full ledger",
+                                    name, lh.getId());
+                        }
                     } else {
-                        log.warn("Error when closing ledger {}. Status={}", lh.getId(), BKException.getMessage(rc));
+                        log.warn("[{}] Error when closing ledger {}, trigger by rollover full ledger, Status={}",
+                                name, lh.getId(), BKException.getMessage(rc));
                     }
 
                     ledgerClosed(lh);
                     createLedgerAfterClosed();
                 }
-            }, System.nanoTime());
+            }, null);
         }
     }
 
@@ -4353,7 +4357,26 @@ public void checkInactiveLedgerAndRollOver() {
         long currentTimeMs = System.currentTimeMillis();
         if (inactiveLedgerRollOverTimeMs > 0 && currentTimeMs > (lastAddEntryTimeMs + inactiveLedgerRollOverTimeMs)) {
             log.info("[{}] Closing inactive ledger, last-add entry {}", name, lastAddEntryTimeMs);
-            ledgerClosed(currentLedger);
+            if (STATE_UPDATER.compareAndSet(this, State.LedgerOpened, State.ClosingLedger)) {
+                LedgerHandle currentLedger = this.currentLedger;
+                currentLedger.asyncClose((rc, lh, o) -> {
+                    checkArgument(currentLedger.getId() == lh.getId(), "ledgerId %s doesn't match with "
+                            + "acked ledgerId %s", currentLedger.getId(), lh.getId());
+
+                    if (rc == BKException.Code.OK) {
+                        if (log.isDebugEnabled()) {
+                            log.debug("[{}] Successfully closed ledger {}, trigger by inactive ledger check",
+                                    name, lh.getId());
+                        }
+                    } else {
+                        log.warn("[{}] Error when closing ledger {}, trigger by inactive ledger check, Status={}",
+                                name, lh.getId(), BKException.getMessage(rc));
+                    }
+
+                    ledgerClosed(lh);
+                    // we do not create ledger here, since topic is inactive for a long time.
+                }, null);
+            }
         }
     }
 
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index dd30cde72e769..70ddbb9998fd8 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -88,6 +88,7 @@
 import org.apache.bookkeeper.client.PulsarMockBookKeeper;
 import org.apache.bookkeeper.client.PulsarMockLedgerHandle;
 import org.apache.bookkeeper.client.api.LedgerEntries;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
 import org.apache.bookkeeper.client.api.ReadHandle;
 import org.apache.bookkeeper.conf.ClientConfiguration;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -3859,12 +3860,26 @@ public void testInactiveLedgerRollOver() throws Exception {
         ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("rollover_inactive", config);
         ManagedCursor cursor = ledger.openCursor("c1");
 
+        List ledgerIds = new ArrayList<>();
+
         int totalAddEntries = 5;
         for (int i = 0; i < totalAddEntries; i++) {
             String content = "entry"; // 5 bytes
             ledger.checkInactiveLedgerAndRollOver();
             ledger.addEntry(content.getBytes());
             Thread.sleep(inactiveLedgerRollOverTimeMs * 5);
+
+            ledgerIds.add(ledger.currentLedger.getId());
+        }
+
+        Map ledgerMap = bkc.getLedgerMap();
+        // skip check last ledger, it should be open
+        for (int i = 0; i < ledgerIds.size() - 1; i++) {
+            long ledgerId = ledgerIds.get(i);
+            LedgerMetadata ledgerMetadata = ledgerMap.get(ledgerId).getLedgerMetadata();
+            if (ledgerMetadata != null) {
+                assertTrue(ledgerMetadata.isClosed());
+            }
         }
 
         List ledgers = ledger.getLedgersInfoAsList();

From ff58c6203ba6367f8ba293bc1a4e994725b5758a Mon Sep 17 00:00:00 2001
From: zhangwd3 
Date: Tue, 16 May 2023 11:23:11 +0800
Subject: [PATCH 280/404] correcting spelling mistakes

Signed-off-by: zhangwd3 
---
 .../org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java  | 2 +-
 .../broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java     | 2 +-
 .../apache/pulsar/client/api/SimpleProducerConsumerTest.java  | 2 +-
 .../apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java  | 2 +-
 .../src/main/java/org/apache/pulsar/client/api/Reader.java    | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
index 1405979ae3b12..17aa7170fc63c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
@@ -161,7 +161,7 @@ public static long getCpuUsageForCGroup() {
      * 
      * 

      * Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this
-     * far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal.
+     * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal.
      */
     public static ResourceUsage getCpuUsageForEntireHost() {
         try (Stream stream = Files.lines(Paths.get(PROC_STAT_PATH))) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
index 5d2b7bdd09adf..2f7ca614943b1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
@@ -155,7 +155,7 @@ private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) {
      * 
      *
      * Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this
-     * far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal.
+     * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal.
      */
     private double getTotalCpuUsageForEntireHost() {
         LinuxInfoUtils.ResourceUsage cpuUsageForEntireHost = getCpuUsageForEntireHost();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 1bc437195d96d..f3a00531eba56 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -3136,7 +3136,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe
                     pulsarClient.newProducer().topic("persistent://my-property/my-ns/myenc-topic1")
                             .addEncryptionKey("client-non-existant-rsa.pem").cryptoKeyReader(new EncKeyReader())
                             .create();
-            Assert.fail("Producer creation should not suceed if failing to read key");
+            Assert.fail("Producer creation should not succeed if failing to read key");
         } catch (Exception e) {
             // ok
         }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java
index 9f94d894632b8..e126d963a88f5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java
@@ -2296,7 +2296,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe
                 .addEncryptionKey("client-non-existant-rsa.pem")
                 .cryptoKeyReader(new EncKeyReader())
                 .create();
-            Assert.fail("Producer creation should not suceed if failing to read key");
+            Assert.fail("Producer creation should not succeed if failing to read key");
         } catch (Exception e) {
             // ok
         }
diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java
index 451b9fa638203..98fcdb453bb76 100644
--- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java
+++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java
@@ -36,14 +36,14 @@ public interface Reader extends Closeable {
     /**
      * Read the next message in the topic.
      *
-     * @return the next messasge
+     * @return the next message
      * @throws PulsarClientException
      */
     Message readNext() throws PulsarClientException;
 
     /**
      * Read the next message in the topic waiting for a maximum of timeout
-     * time units. Returns null if no message is recieved in that time.
+     * time units. Returns null if no message is received in that time.
      *
      * @return the next message(Could be null if none received in time)
      * @throws PulsarClientException

From 9d7609f0de1468b68209875fe44527514c1f92de Mon Sep 17 00:00:00 2001
From: Kengo Seki 
Date: Tue, 16 May 2023 23:35:04 +0900
Subject: [PATCH 281/404] [improve][io] Upgrade Hadoop and HBase versions
 (#20008)

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 55f0224991759..83aba7682eee0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -200,8 +200,8 @@ flexible messaging model and an intuitive client API.
     0.11.1
     0.28.0
     2.10.2
-    3.3.4
-    2.4.15
+    3.3.5
+    2.4.16
     31.0.1-jre
     1.0
     0.16.1

From 1a7a876634f32d5d9218977dc83f8ae08a08bf57 Mon Sep 17 00:00:00 2001
From: "Kim, Joo Hyuk" 
Date: Wed, 17 May 2023 16:05:01 +0900
Subject: [PATCH 282/404] [improve][client] Document Java Client's Seek logic
 thread-safety improved in #20242 (#20284)

---
 .../org/apache/pulsar/client/api/Consumer.java   | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java
index 88ad24fe1f484..c67ad08c83631 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java
@@ -465,6 +465,9 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message,
 
     /**
      * Reset the subscription associated with this consumer to a specific message id.
+     * 

+     * If there is already a seek operation in progress, the method will log a warning and
+     * return a future completed exceptionally.
      *
      * 
The message id can either be a specific message or represent the first or last messages in the topic.
      * 
    
@@ -483,6 +486,9 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message,
 
     /**
      * Reset the subscription associated with this consumer to a specific message publish time.
+     * 
    
+     * If there is already a seek operation in progress, the method will log a warning and
+     * return a future completed exceptionally.
      *
      * @param timestamp
      *            the message publish time where to reposition the subscription
@@ -493,6 +499,10 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message,
     /**
      * Reset the subscription associated with this consumer to a specific message ID or message publish time.
      * 
    
+     * If there is already a seek operation in progress, the method will log a warning and
+     * return a future completed exceptionally.
+     *
+     * 
    
      * The Function input is topic+partition. It returns only timestamp or MessageId.
      * 
    
      * The return value is the seek position/timestamp of the current partition.
@@ -523,11 +533,17 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message,
 
     /**
      * The asynchronous version of {@link Consumer#seek(MessageId)}.
+     * 
    
+     * If there is already a seek operation in progress, the method will log a warning and
+     * return a future completed exceptionally.
      */
     CompletableFuture seekAsync(MessageId messageId);
 
     /**
      * Reset the subscription associated with this consumer to a specific message publish time.
+     * 
    
+     * If there is already a seek operation in progress, the method will log a warning and
+     * return a future completed exceptionally.
      *
      * @param timestamp
      *            the message publish time where to reposition the subscription

From c7a4060763b905925ffe8e9690ea983d863aa52f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 17 May 2023 04:14:44 -0500
Subject: [PATCH 283/404] [feat][meta] Upgrade to jetcd to 0.7.5 (#20339)

---
 .../server/src/assemble/LICENSE.bin.txt       |  8 ++--
 pom.xml                                       |  2 +-
 .../metadata/BaseMetadataStoreTest.java       |  7 +--
 .../impl/LeaderElectionImplTest.java          |  2 +-
 .../metadata/impl/EtcdMetadataStoreTest.java  | 17 ++++---
 pulsar-sql/presto-distribution/LICENSE        | 15 +++----
 src/owasp-dependency-check-suppressions.xml   | 45 -------------------
 7 files changed, 28 insertions(+), 68 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 993e0997e53ee..220f0ac0758b8 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -433,7 +433,6 @@ The Apache Software License, Version 2.0
     - io.grpc-grpc-services-1.45.1.jar
     - io.grpc-grpc-xds-1.45.1.jar
     - io.grpc-grpc-rls-1.45.1.jar
-    - com.google.auto.service-auto-service-annotations-1.0.jar
   * Perfmark
     - io.perfmark-perfmark-api-0.19.0.jar
   * OpenCensus
@@ -475,6 +474,7 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-core-4.3.8.jar
     - io.vertx-vertx-web-4.3.8.jar
     - io.vertx-vertx-web-common-4.3.8.jar
+    - io.vertx-vertx-grpc-4.3.5.jar
   * Apache ZooKeeper
     - org.apache.zookeeper-zookeeper-3.8.1.jar
     - org.apache.zookeeper-zookeeper-jute-3.8.1.jar
@@ -487,8 +487,10 @@ The Apache Software License, Version 2.0
     - com.google.auto.value-auto-value-annotations-1.9.jar
     - com.google.re2j-re2j-1.5.jar
   * Jetcd
-    - io.etcd-jetcd-common-0.5.11.jar
-    - io.etcd-jetcd-core-0.5.11.jar
+    - io.etcd-jetcd-api-0.7.5.jar
+    - io.etcd-jetcd-common-0.7.5.jar
+    - io.etcd-jetcd-core-0.7.5.jar
+    - io.etcd-jetcd-grpc-0.7.5.jar
   * IPAddress
     - com.github.seancfoley-ipaddress-5.3.3.jar
   * RxJava
diff --git a/pom.xml b/pom.xml
index 83aba7682eee0..75c25e5478893 100644
--- a/pom.xml
+++ b/pom.xml
@@ -242,7 +242,7 @@ flexible messaging model and an intuitive client API.
     5.3.27
     4.5.13
     4.4.15
-    0.5.11
+    0.7.5
     2.0
     1.10.12
     5.3.3
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
index 57a4e388572fa..ec6e6e03eae71 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
@@ -21,7 +21,7 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import io.etcd.jetcd.launcher.EtcdCluster;
-import io.etcd.jetcd.launcher.EtcdClusterFactory;
+import io.etcd.jetcd.test.EtcdClusterExtension;
 import java.io.File;
 import java.net.URI;
 import java.util.UUID;
@@ -84,10 +84,11 @@ public Object[][] implementations() {
 
     private synchronized String getEtcdClusterConnectString() {
         if (etcdCluster == null) {
-            etcdCluster = EtcdClusterFactory.buildCluster("test", 1, false);
+            etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build()
+                    .cluster();
             etcdCluster.start();
         }
-        return etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
+        return etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
     }
 
     public static Supplier stringSupplier(Supplier supplier) {
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java
index 027521d2ffc17..09c9d71c41a30 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java
@@ -29,7 +29,7 @@
 
 public class LeaderElectionImplTest extends BaseMetadataStoreTest {
 
-    @Test(dataProvider = "impl", timeOut = 10000)
+    @Test(dataProvider = "impl", timeOut = 20000)
     public void validateDeadLock(String provider, Supplier urlSupplier)
             throws Exception {
         if (provider.equals("Memory") || provider.equals("RocksDB")) {
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java
index f1a1b5626acce..bdcd0614d0375 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java
@@ -23,8 +23,8 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.google.common.io.Resources;
 import io.etcd.jetcd.launcher.EtcdCluster;
-import io.etcd.jetcd.launcher.EtcdClusterFactory;
 
+import io.etcd.jetcd.test.EtcdClusterExtension;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -44,7 +44,8 @@ public class EtcdMetadataStoreTest {
     @Test
     public void testCluster() throws Exception {
         @Cleanup
-        EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-cluster", 3, false);
+        EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-cluster").withNodes(3)
+                .withSsl(false).build().cluster();
         etcdCluster.start();
 
         EtcdConfig etcdConfig = EtcdConfig.builder().useTls(false)
@@ -56,7 +57,7 @@ public void testCluster() throws Exception {
         new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig);
 
         String metadataURL =
-                "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
+                "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
 
         @Cleanup
         MetadataStore store = MetadataStoreFactory.create(metadataURL,
@@ -71,7 +72,8 @@ public void testCluster() throws Exception {
     @Test
     public void testClusterWithTls() throws Exception {
         @Cleanup
-        EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-cluster", 3, true);
+        EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-cluster").withNodes(3)
+                .withSsl(true).build().cluster();
         etcdCluster.start();
 
         EtcdConfig etcdConfig = EtcdConfig.builder().useTls(true)
@@ -86,7 +88,7 @@ public void testClusterWithTls() throws Exception {
         new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig);
 
         String metadataURL =
-                "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
+                "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
 
         @Cleanup
         MetadataStore store = MetadataStoreFactory.create(metadataURL,
@@ -101,7 +103,8 @@ public void testClusterWithTls() throws Exception {
     @Test
     public void testTlsInstance() throws Exception {
         @Cleanup
-        EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-tls", 1, true);
+        EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-tls").withNodes(1)
+                .withSsl(true).build().cluster();
         etcdCluster.start();
 
         EtcdConfig etcdConfig = EtcdConfig.builder().useTls(true)
@@ -115,7 +118,7 @@ public void testTlsInstance() throws Exception {
         new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig);
 
         String metadataURL =
-                "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
+                "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","));
 
         @Cleanup
         MetadataStore store = MetadataStoreFactory.create(metadataURL,
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 09d1396b70419..a85d1bc363c1b 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -272,9 +272,13 @@ The Apache Software License, Version 2.0
     - grpc-protobuf-lite-1.45.1.jar
     - grpc-stub-1.45.1.jar
   * JEtcd
-    - jetcd-common-0.5.11.jar
-    - jetcd-core-0.5.11.jar
-
+    - jetcd-api-0.7.5.jar
+    - jetcd-common-0.7.5.jar
+    - jetcd-core-0.7.5.jar
+    - jetcd-grpc-0.7.5.jar
+  * Vertx
+    - vertx-core-4.3.8.jar
+    - vertx-grpc-4.3.5.jar
  * Joda Time
     - joda-time-2.10.10.jar
     - failsafe-2.4.4.jar
@@ -476,8 +480,6 @@ The Apache Software License, Version 2.0
     - swagger-annotations-1.6.2.jar
   * Perfmark
     - perfmark-api-0.19.0.jar
-  * Annotations
-    - auto-service-annotations-1.0.jar
   * RabbitMQ Java Client
     - amqp-client-5.5.3.jar
   * Stream Lib
@@ -520,9 +522,6 @@ MIT License
    - jcl-over-slf4j-1.7.32.jar
  * Checker Qual
    - checker-qual-3.12.0.jar
- * Annotations
-   - animal-sniffer-annotations-1.19.jar
-   - annotations-4.1.1.4.jar
  * ScribeJava
    - scribejava-apis-6.9.0.jar
    - scribejava-core-6.9.0.jar
diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml
index d599f2de3e85a..a59b3b999e2bf 100644
--- a/src/owasp-dependency-check-suppressions.xml
+++ b/src/owasp-dependency-check-suppressions.xml
@@ -229,51 +229,6 @@
         CVE-2021-42550
     
 
-    
-    
-        
-        c85851ca3ea8128d480d3f75c568a37e64e8a77b
-        CVE-2020-15106
-    
-    
-        
-        c85851ca3ea8128d480d3f75c568a37e64e8a77b
-        CVE-2020-15112
-    
-    
-        
-        c85851ca3ea8128d480d3f75c568a37e64e8a77b
-        CVE-2020-15113
-    
-
-    
-        
-        6dac6efe035a2be9ba299fbf31be5f903401869f
-        CVE-2020-15106
-    
-    
-        
-        6dac6efe035a2be9ba299fbf31be5f903401869f
-        CVE-2020-15112
-    
-    
-        
-        6dac6efe035a2be9ba299fbf31be5f903401869f
-        CVE-2020-15113
-    
-
     
     
         
Date: Wed, 17 May 2023 19:43:42 +0800
Subject: [PATCH 284/404] [fix][doc] Correcting spelling mistakes (#20340)

Signed-off-by: zhangwd3 
Co-authored-by: tison 
---
 .../java/org/apache/pulsar/client/impl/schema/SchemaUtils.java  | 2 +-
 .../main/java/org/apache/pulsar/common/api/raw/RawMessage.java  | 2 +-
 .../pulsar/functions/instance/state/BKStateStoreImpl.java       | 2 +-
 .../functions/auth/KubernetesSecretsTokenAuthProvider.java      | 2 +-
 .../apache/pulsar/functions/worker/rest/api/ComponentImpl.java  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java
index 526b0c96be05b..8acbf26559b7b 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java
@@ -210,7 +210,7 @@ public static String jsonifySchemaInfo(SchemaInfo schemaInfo) {
     }
 
     /**
-     * Jsonify the schema info with verison.
+     * Jsonify the schema info with version.
      *
      * @param schemaInfoWithVersion the schema info
      * @return the jsonified schema info with version
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java
index 8e2f51c4edbf0..a02208396fc91 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java
@@ -102,7 +102,7 @@ public interface RawMessage {
     Optional getKey();
 
     /**
-     * Get the schema verison of the message.
+     * Get the schema version of the message.
      *
      * @return the schema version of the message
      */
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java
index df1eae9b6a7ab..bf43f18b175e7 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java
@@ -166,7 +166,7 @@ public CompletableFuture getAsync(String key) {
                             data.readBytes(result);
                             // Set position to off the buffer to the beginning, since the position after the
                             // read is going to be end of the buffer
-                            // If we do not rewind to the begining here, users will have to explicitly do
+                            // If we do not rewind to the beginning here, users will have to explicitly do
                             // this in their function code
                             // in order to use any of the ByteBuffer operations
                             result.position(0);
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java
index e0543f28ac160..913171aca9bc8 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java
@@ -161,7 +161,7 @@ public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional
Date: Wed, 17 May 2023 10:29:45 -0500
Subject: [PATCH 285/404] [cleanup] Consolidate certs used in tests (#20336)

Builds on: https://github.com/apache/pulsar/pull/20289

### Motivation

There are many certificates in our test code base. It would be much simpler to have one place were we create and manage certificates so that when we need to make changes, they are consolidated.

There is likely one or two more PRs to finish consolidating certs.

### Modifications

* Remove certs that are no longer used
* Replace references to old certs with references to the `certificate-authority` certs
* Create new server certs with valid hostnames on them so that tests will pass. Document the process used to create these certs.
* Fix an issue in the `PulsarTestContext` where the configuration was not correctly updated.
* Remove configurations that allow for insecure connections in tests that are doing some kind of TLS verification. The only places where we leave insecure validation in place is tests that are specifically verifying the functionality.
* Copy `certificate-authority` to the relevant `bouncy-castle` directory

### Verifying this change

When tests pass, this change will be correctly verified.

### Documentation

- [x] `doc`
This PR includes doc changes

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/48
---
 bouncy-castle/bcfips-include-test/pom.xml     |  22 +++
 .../client/TlsProducerConsumerBase.java       |  23 ++-
 .../authentication/tls/broker-cert.pem        |  71 ----------
 .../authentication/tls/broker-key.pem         |  28 ----
 .../resources/authentication/tls/cacert.pem   |  78 ----------
 .../authentication/tls/client-cert.pem        |  71 ----------
 .../authentication/tls/client-key.pem         |  28 ----
 build/regenerate_certs_for_tests.sh           |   7 -
 .../admin/BrokerAdminClientTlsAuthTest.java   |   2 +-
 .../broker/testcontext/PulsarTestContext.java |   3 +
 .../AuthenticatedProducerConsumerTest.java    |  53 ++++---
 ...enticationTlsHostnameVerificationTest.java |  26 ++--
 .../api/ClientAuthenticationTlsTest.java      |  27 ++--
 .../client/api/ProducerConsumerBase.java      |   5 -
 .../pulsar/client/api/ProxyProtocolTest.java  |  12 +-
 .../client/api/TlsHostVerificationTest.java   |  36 +++--
 .../client/api/TlsProducerConsumerBase.java   |  23 ++-
 .../client/api/TlsProducerConsumerTest.java   |  20 +--
 .../apache/pulsar/client/api/TlsSniTest.java  |   6 +-
 .../TokenExpirationProduceConsumerTest.java   |  10 +-
 .../worker/PulsarFunctionLocalRunTest.java    |  16 ++-
 .../worker/PulsarFunctionPublishTest.java     |  16 ++-
 .../pulsar/io/AbstractPulsarE2ETest.java      |  16 ++-
 .../pulsar/io/PulsarFunctionAdminTest.java    |  20 +--
 .../pulsar/io/PulsarFunctionTlsTest.java      |  21 ++-
 .../proxy/ProxyPublishConsumeTlsTest.java     |  13 +-
 tests/certificate-authority/.gitignore        |   3 +
 tests/certificate-authority/README.md         |  24 +++-
 tests/certificate-authority/index.txt         |   2 +
 tests/certificate-authority/newcerts/1007.pem | 111 +++++++++++++++
 tests/certificate-authority/newcerts/1008.pem | 110 ++++++++++++++
 tests/certificate-authority/openssl.cnf       |  17 ++-
 tests/certificate-authority/serial            |   2 +-
 .../server-keys/broker.cert.pem               | 134 ++++++++++++++----
 .../server-keys/broker.csr.pem                |  26 ++--
 .../server-keys/broker.key-pk8.pem            |  52 +++----
 .../server-keys/broker.key.pem                |  50 +++----
 .../server-keys/proxy.cert.pem                | 133 +++++++++++++----
 .../server-keys/proxy.csr.pem                 |  26 ++--
 .../server-keys/proxy.key-pk8.pem             |  52 +++----
 .../server-keys/proxy.key.pem                 |  50 +++----
 41 files changed, 812 insertions(+), 633 deletions(-)
 delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem
 delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem
 delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem
 delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem
 delete mode 100644 bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem
 create mode 100644 tests/certificate-authority/.gitignore
 create mode 100644 tests/certificate-authority/newcerts/1007.pem
 create mode 100644 tests/certificate-authority/newcerts/1008.pem

diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml
index 3b8c6754c3fa9..e8348be9292cd 100644
--- a/bouncy-castle/bcfips-include-test/pom.xml
+++ b/bouncy-castle/bcfips-include-test/pom.xml
@@ -85,6 +85,28 @@
           true
         
       
+      
+        maven-resources-plugin
+        
+          
+            copy-resources
+            test-compile
+            
+              copy-resources
+            
+            
+              ${project.build.testOutputDirectory}/certificate-authority
+              true
+              
+                
+                  ${project.parent.parent.basedir}/tests/certificate-authority
+                  false
+                
+              
+            
+          
+        
+      
     
   
 
diff --git a/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java b/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java
index 330d4fbc06897..e8e12838defef 100644
--- a/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java
+++ b/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java
@@ -37,11 +37,6 @@
 import org.testng.annotations.BeforeMethod;
 
 public class TlsProducerConsumerBase extends ProducerConsumerBase {
-    protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
     private final String clusterName = "use";
 
     @BeforeMethod(alwaysRun = true)
@@ -63,9 +58,9 @@ protected void cleanup() throws Exception {
     protected void internalSetUpForBroker() throws Exception {
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setClusterName(clusterName);
         conf.setTlsRequireTrustedClientCertOnConnect(true);
         Set tlsProtocols = Sets.newConcurrentHashSet();
@@ -81,12 +76,12 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl)
         }
 
         ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(lookupUrl)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
                 .operationTimeout(1000, TimeUnit.MILLISECONDS);
         if (addCertificates) {
             Map authParams = new HashMap<>();
-            authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-            authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+            authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+            authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
             clientBuilder.authentication(AuthenticationTls.class.getName(), authParams);
         }
         pulsarClient = clientBuilder.build();
@@ -94,15 +89,15 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl)
 
     protected void internalSetUpForNamespace() throws Exception {
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         if (admin != null) {
             admin.close();
         }
 
         admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false)
                 .authentication(AuthenticationTls.class.getName(), authParams).build());
         admin.clusters().createCluster(clusterName, ClusterData.builder()
                 .serviceUrl(brokerUrl.toString())
diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem
deleted file mode 100644
index e2b44e0bf0c42..0000000000000
--- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem
+++ /dev/null
@@ -1,71 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 15537474201172114493 (0xd7a0327703a8fc3d)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: CN=CARoot
-        Validity
-            Not Before: Feb 22 06:26:33 2023 GMT
-            Not After : Feb 19 06:26:33 2033 GMT
-        Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=localhost
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:af:bf:b7:2d:98:ad:9d:f6:da:a3:13:d4:62:0f:
-                    98:be:1c:a2:89:22:ba:6f:d5:fd:1f:67:e3:91:03:
-                    98:80:81:0e:ed:d8:f6:70:7f:2c:36:68:3d:53:ea:
-                    58:3a:a6:d5:89:66:4b:bd:1e:57:71:13:6d:4b:11:
-                    e5:40:a5:76:84:24:92:40:58:80:96:c9:1f:2c:c4:
-                    55:eb:a3:79:73:70:5c:37:9a:89:ed:2f:ba:6b:e3:
-                    82:7c:69:4a:02:54:8b:81:5e:3c:bf:4c:8a:cb:ea:
-                    2c:5e:83:e7:b7:10:08:5f:82:58:a3:89:d1:da:92:
-                    ba:2a:28:ee:30:28:3f:5b:ae:10:71:96:c7:e1:12:
-                    c5:b0:1a:ad:44:6f:44:3a:11:4a:9a:3c:0f:8d:06:
-                    80:7b:34:ef:3f:6c:f4:5e:c5:44:54:1e:c8:dd:c7:
-                    80:85:80:d9:68:e6:c6:53:03:77:e1:fe:18:61:07:
-                    77:05:4c:ed:59:bc:5d:41:38:6a:ef:5d:a1:b2:60:
-                    98:d4:48:28:95:02:8a:0e:fd:cf:7b:1b:d2:11:cc:
-                    10:0c:50:73:d7:cc:38:6c:83:dd:79:26:aa:90:c8:
-                    9b:84:86:bc:59:e9:62:69:f4:98:1b:c4:80:78:7e:
-                    a0:1a:81:9d:d2:e1:66:dd:c4:cc:fc:63:04:ac:ec:
-                    a7:35
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Subject Alternative Name: 
-                DNS:localhost, IP Address:127.0.0.1
-    Signature Algorithm: sha256WithRSAEncryption
-         5f:e0:73:7b:5e:db:c0:8b:5e:4c:43:5f:80:94:ca:0b:f8:e9:
-         9b:93:91:3d:b1:3a:99:ce:1c:fb:15:32:68:3e:b9:9c:52:d0:
-         4b:7f:17:09:ec:af:6b:05:3e:e2:a3:e6:cc:bb:53:d7:ea:4a:
-         82:3c:4e:a5:37:ca:f4:1e:38:e2:d6:a5:98:4d:ee:b9:e2:9a:
-         48:d2:9f:0a:bc:61:42:70:22:b9:fb:cd:73:72:fb:94:13:ac:
-         6e:c5:b6:4b:24:ef:0f:df:2d:e6:56:da:b2:76:e8:16:be:7f:
-         3f:1b:99:6e:32:3e:b9:f4:2b:35:72:c7:e4:c6:a5:92:68:c0:
-         1f:a0:f7:17:fd:a3:b6:73:98:d3:ea:1c:af:ea:7d:f8:a0:27:
-         40:dc:4e:8b:13:28:ba:65:60:c5:90:57:e8:54:c1:83:b4:9d:
-         f0:ae:2a:de:27:57:e5:a2:e5:f4:87:1c:df:6b:dc:7b:43:ff:
-         b6:be:0b:3b:b2:8b:1a:36:dc:e3:57:aa:52:ef:23:d6:50:d7:
-         e4:72:8f:a0:0a:43:de:3d:f2:42:5b:fa:ed:1f:8d:0e:cf:c5:
-         6a:ce:3b:8e:fd:6b:68:01:a9:f9:d2:0e:0d:ac:39:8d:f5:6c:
-         80:f8:49:af:bb:b9:d4:81:b9:f3:b2:b6:ce:75:1c:20:e8:6a:
-         53:dc:26:86
------BEGIN CERTIFICATE-----
-MIIDCTCCAfGgAwIBAgIJANegMncDqPw9MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV
-BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ
-BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL
-Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3
-DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv1f0fZ+OR
-A5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICWyR8sxFXr
-o3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHakroqKO4w
-KD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CFgNlo5sZT
-A3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPXzDhsg915
-JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAGjHjAcMBoG
-A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAX+Bz
-e17bwIteTENfgJTKC/jpm5ORPbE6mc4c+xUyaD65nFLQS38XCeyvawU+4qPmzLtT
-1+pKgjxOpTfK9B444talmE3uueKaSNKfCrxhQnAiufvNc3L7lBOsbsW2SyTvD98t
-5lbasnboFr5/PxuZbjI+ufQrNXLH5MalkmjAH6D3F/2jtnOY0+ocr+p9+KAnQNxO
-ixMoumVgxZBX6FTBg7Sd8K4q3idX5aLl9Icc32vce0P/tr4LO7KLGjbc41eqUu8j
-1lDX5HKPoApD3j3yQlv67R+NDs/Fas47jv1raAGp+dIODaw5jfVsgPhJr7u51IG5
-87K2znUcIOhqU9wmhg==
------END CERTIFICATE-----
diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem
deleted file mode 100644
index 004bf8e21a7a9..0000000000000
--- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvv7ctmK2d9tqj
-E9RiD5i+HKKJIrpv1f0fZ+ORA5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21L
-EeVApXaEJJJAWICWyR8sxFXro3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixe
-g+e3EAhfglijidHakroqKO4wKD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/
-bPRexURUHsjdx4CFgNlo5sZTA3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO
-/c97G9IRzBAMUHPXzDhsg915JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8
-YwSs7Kc1AgMBAAECggEAAaWEK9MwXTiA1+JJrRmETtOp2isPIBkbI/4vLZ6hASM0
-ZpoPxQIMAf58BJs/dF03xu/EaeMs4oxSC9ABG9fxAk/tZtjta3w65Ip6W5jOfHxj
-AMpb3HMEBhq9kDjUTq1IGVAutYQcEMkC3WfS9e4ahfqMpguWgbu6LsbvZFgcL9mv
-pGnKv9YVe6Xk6isvqtq6G1af0rd7c//xF0i0e/qEo83Buok3gLEZOELZbcRxjUYc
-jnyglnXnwkGjuL4E3wgS3l73ZKsb6+AYoqhMPVz8t4/PN3tTrsBJKOSYo8KzIm0U
-ek9T8XmPbP0cuheRxp9Dp8TXJJQZK0N9jz+EL0ogQQKBgQDnavm8GpR4pap9cDOc
-+YI5s823b507pNdSU8elO9gLsP0JlFzv+sqghVko29r85D7Vn3MkgYTy0S4ANLCs
-0NFDY8N2QH6U1dTkk1QXZydVZDuKJ5SSpC4v+Vafl8yDxhB4Nlxhbm9vJEMfLcXh
-2kL6UlAuFDtYD0AdczwnHu5DjQKBgQDCauocm55FpcyDMMBO2CjurxcjBYS3S1xT
-Bz+sPtxJLjlKbAt8kSHUQcCcX9zhrQBfsT38LATCmKaOFqUW5/PPh2LcrxiMqlL1
-OJBUJ3Te2LTjlUn8r+DHv/69UIh5tchwRr3YgB0DuIs7jfmr4VfiOWTBtPVhoGFR
-1Wt60j30SQKBgHzreS26J2VNAFBALgxRf6OIVMbtgDG/FOCDCyU9vazp+F2gcd61
-QYYPFYcBzx9uUiDctroBFHRCyJMh3jEbc6ruAogl3m6XUxmkEeOkMk5dEerM3N2f
-tLL+5Gy385U6aI+LwKhzhcG4EGeXPNdjC362ykNldnddnB2Jo/H2N2XNAoGAdnft
-xpbxP+GDGKIZXTIM5zzcLWQMdiC+1n1BSHVZiGJZWMczzKknYw7aDq+/iekApE79
-xW8RS373ZvfXi3i2Mcx+6pjrrbOQL4tTL2SHq8+DknaDCi4mG7IbyUKMlxW1WO1S
-e929UGogtZ6S+DCte9WbVwosyFuRUetpvgLk67kCgYBWetihZjgBWrqVYT24TTRH
-KxzSzH1JgzzF9qgTdlhXDv9hC+Kc0uTKsgViesDqVuCOjkwzY5OQr9c6duO0fwwP
-qNk/qltdgjMC5iiv7duyukfbEuqKEdGGer9HFb7en96dZdVQJpYHaaslAGurtD80
-ejCQZgzR2XaHSuIQb0IUVQ==
------END PRIVATE KEY-----
diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem
deleted file mode 100644
index 4ed454ec52a52..0000000000000
--- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem
+++ /dev/null
@@ -1,78 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 15358526754272834781 (0xd52472b5c5c3f4dd)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: CN=CARoot
-        Validity
-            Not Before: Feb 22 06:26:32 2023 GMT
-            Not After : Feb 19 06:26:32 2033 GMT
-        Subject: CN=CARoot
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:d0:87:45:0b:b4:83:11:ab:5a:b4:b6:1c:15:d4:
-                    92:6a:0c:ac:3b:76:da:ff:8d:61:1b:bd:96:bd:d7:
-                    b0:70:23:87:d4:00:19:b2:e5:63:b7:80:58:4a:a4:
-                    d8:a8:a6:4f:eb:c8:8c:54:07:f5:56:52:23:64:fc:
-                    66:54:39:f1:33:d0:e5:cc:b6:40:c8:d7:9a:9f:0e:
-                    c4:aa:57:b0:b3:e2:41:61:54:ca:1f:90:3b:18:ef:
-                    60:d2:dc:ee:34:29:33:08:1b:37:4b:c4:ca:7e:cb:
-                    94:7f:50:c4:8d:16:2f:90:03:94:07:bf:cf:52:ff:
-                    24:54:56:ac:74:6c:d3:31:8c:ce:ef:b3:14:5a:5b:
-                    8a:0c:83:2d:e1:f7:4d:60:2f:a1:4d:85:38:96:7f:
-                    01:2f:9a:99:c7:2e:3d:09:4d:5e:53:df:fd:29:9f:
-                    ff:6b:e4:c2:a1:e3:67:85:db:e2:02:4d:6f:29:d4:
-                    e1:b3:a2:34:71:e0:90:dd:3f:b3:3f:86:41:8c:97:
-                    09:e6:c3:de:a0:0e:d3:d4:3e:ce:ea:58:70:e6:9f:
-                    24:a8:19:ca:df:61:b8:9c:c3:4e:53:d0:69:96:44:
-                    84:76:2b:99:65:08:06:42:d4:b2:76:a7:2f:69:12:
-                    d5:c2:65:a6:ff:2c:77:73:00:e7:97:a5:77:6b:8a:
-                    9c:3f
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Basic Constraints: critical
-                CA:TRUE
-            X509v3 Subject Key Identifier: 
-                A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69
-            X509v3 Authority Key Identifier: 
-                keyid:A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69
-                DirName:/CN=CARoot
-                serial:D5:24:72:B5:C5:C3:F4:DD
-
-    Signature Algorithm: sha256WithRSAEncryption
-         21:b1:4d:2b:14:1e:5a:91:5d:28:9e:ba:cb:ed:f1:96:da:c3:
-         fa:8d:b5:74:e4:c5:fb:2f:3e:39:b4:a6:59:69:dd:84:64:a8:
-         f0:e0:39:d2:ef:87:cc:8b:09:9f:0a:84:1f:d0:96:9c:4b:64:
-         ea:08:09:26:1c:84:f4:06:5f:5e:b9:ba:b3:3c:6c:81:e0:93:
-         46:89:07:51:95:36:77:96:76:5d:a6:68:71:bb:60:88:a7:83:
-         27:7c:66:5d:64:36:cb:8e:bd:02:f7:fb:52:63:83:2f:fe:57:
-         4c:d5:0c:1b:ea:ef:88:ad:8c:a9:d4:b3:2c:b8:c4:e2:90:cb:
-         0f:24:0e:df:fc:2a:c6:83:08:49:45:b0:41:85:0e:b4:6f:f7:
-         18:56:7b:a5:0b:f6:1b:7f:72:88:ee:c8:ef:b3:e3:3e:f0:68:
-         1b:c9:55:bb:4d:21:65:6b:9e:5c:dd:60:4b:7f:f1:84:f8:67:
-         51:c2:60:88:42:6e:6c:9c:14:b8:96:b0:18:10:97:2c:94:e7:
-         79:14:7b:d1:a2:a4:d8:94:84:ac:a9:ca:17:95:c2:27:8b:2b:
-         d8:19:6a:14:4b:c3:03:a6:30:55:40:bd:ce:0c:c2:d5:af:7d:
-         6d:65:89:6b:74:ed:21:12:f1:aa:c9:c9:ba:da:9a:ca:14:6c:
-         39:f4:02:32
------BEGIN CERTIFICATE-----
-MIIDGjCCAgKgAwIBAgIJANUkcrXFw/TdMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV
-BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN
-BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANCH
-RQu0gxGrWrS2HBXUkmoMrDt22v+NYRu9lr3XsHAjh9QAGbLlY7eAWEqk2KimT+vI
-jFQH9VZSI2T8ZlQ58TPQ5cy2QMjXmp8OxKpXsLPiQWFUyh+QOxjvYNLc7jQpMwgb
-N0vEyn7LlH9QxI0WL5ADlAe/z1L/JFRWrHRs0zGMzu+zFFpbigyDLeH3TWAvoU2F
-OJZ/AS+amccuPQlNXlPf/Smf/2vkwqHjZ4Xb4gJNbynU4bOiNHHgkN0/sz+GQYyX
-CebD3qAO09Q+zupYcOafJKgZyt9huJzDTlPQaZZEhHYrmWUIBkLUsnanL2kS1cJl
-pv8sd3MA55eld2uKnD8CAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
-FgQUp1VrURB1zk5bC2T/qW0j+1eIWWkwQQYDVR0jBDowOIAUp1VrURB1zk5bC2T/
-qW0j+1eIWWmhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANUkcrXFw/TdMA0GCSqG
-SIb3DQEBCwUAA4IBAQAhsU0rFB5akV0onrrL7fGW2sP6jbV05MX7Lz45tKZZad2E
-ZKjw4DnS74fMiwmfCoQf0JacS2TqCAkmHIT0Bl9eubqzPGyB4JNGiQdRlTZ3lnZd
-pmhxu2CIp4MnfGZdZDbLjr0C9/tSY4Mv/ldM1Qwb6u+IrYyp1LMsuMTikMsPJA7f
-/CrGgwhJRbBBhQ60b/cYVnulC/Ybf3KI7sjvs+M+8GgbyVW7TSFla55c3WBLf/GE
-+GdRwmCIQm5snBS4lrAYEJcslOd5FHvRoqTYlISsqcoXlcIniyvYGWoUS8MDpjBV
-QL3ODMLVr31tZYlrdO0hEvGqycm62prKFGw59AIy
------END CERTIFICATE-----
diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem
deleted file mode 100644
index 3cf236c401255..0000000000000
--- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem
+++ /dev/null
@@ -1,71 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 15537474201172114494 (0xd7a0327703a8fc3e)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: CN=CARoot
-        Validity
-            Not Before: Feb 22 06:26:33 2023 GMT
-            Not After : Feb 19 06:26:33 2033 GMT
-        Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=superUser
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:cd:43:7d:98:40:f9:b0:5b:bc:ae:db:c0:0b:ad:
-                    26:90:96:e0:62:38:ed:68:b1:70:46:3b:de:44:f9:
-                    14:51:86:10:eb:ca:90:e7:88:e8:f9:91:85:e0:dd:
-                    b5:b4:14:b9:78:e3:86:d5:54:6d:68:ec:14:92:b4:
-                    f8:22:5b:05:3d:ed:31:25:65:08:05:84:ca:e6:0c:
-                    21:12:58:32:c7:1a:60:a3:4f:d2:4a:9e:28:19:7c:
-                    45:84:00:8c:89:dc:de:8a:e5:4f:88:91:cc:a4:f1:
-                    81:45:4c:7d:c2:ff:e2:c1:89:c6:12:73:95:e2:36:
-                    bd:db:ae:8b:5a:68:6a:90:51:de:2b:88:5f:aa:67:
-                    f4:a8:e3:63:dc:be:19:82:cc:9d:7f:e6:8d:fb:82:
-                    be:22:01:3d:56:13:3b:5b:04:b4:e8:c5:18:e6:2e:
-                    0d:fa:ba:4a:8d:e8:c6:5a:a1:51:9a:4a:62:d7:af:
-                    dd:b4:fc:e2:d5:cd:ae:99:6c:5c:61:56:0b:d7:0c:
-                    1a:77:5c:f5:3a:6a:54:b5:9e:33:ac:a9:75:28:9a:
-                    76:af:d0:7a:57:00:1b:91:13:31:fd:42:88:21:47:
-                    05:10:01:2f:59:bb:c7:3a:d9:e1:58:4c:1b:6c:71:
-                    b6:98:ef:dd:03:82:58:a3:32:dc:90:a1:b6:a6:1e:
-                    e1:0b
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Subject Alternative Name: 
-                DNS:localhost, IP Address:127.0.0.1
-    Signature Algorithm: sha256WithRSAEncryption
-         b8:fc:d3:8f:8a:e0:6b:74:57:e2:a3:79:b2:18:60:0b:2c:05:
-         f9:e3:ae:dd:e9:ad:52:88:52:73:b4:12:b0:39:90:65:12:f5:
-         95:0e:5f:4b:f2:06:4a:57:ab:e1:f9:b1:34:68:83:d7:d7:5e:
-         69:0a:16:44:ea:1d:97:53:51:10:51:8b:ec:0a:b3:c8:a3:3d:
-         85:4d:f4:8f:7d:b3:b5:72:e4:9e:d7:f3:01:bf:66:e1:40:92:
-         54:63:16:b6:b5:66:ed:30:38:94:1d:1a:8f:28:34:27:ab:c9:
-         5f:d5:16:7e:e4:f5:93:d2:19:35:44:0a:c4:2e:6a:25:38:1d:
-         ee:5a:c8:29:fa:96:dc:95:82:38:9e:36:3a:68:34:7b:4e:d9:
-         fa:0d:b2:88:a2:6c:4f:03:18:a7:e3:41:67:38:de:e5:f6:ff:
-         2a:1c:f0:ec:1a:02:a7:e8:4e:3a:c3:04:72:f8:6a:4f:28:a6:
-         cf:0b:a2:db:33:74:d1:10:9e:ec:b4:ac:f8:b1:24:f4:ef:0e:
-         05:e4:9d:1b:9a:40:f7:09:66:9c:9d:86:8b:76:96:46:e8:d1:
-         dc:10:c7:7d:0b:69:41:dc:a7:8e:e3:a3:36:e3:42:63:93:8c:
-         91:80:0d:27:11:1c:2d:ae:fb:92:88:6c:6b:09:40:1a:30:dd:
-         8f:ac:0f:62
------BEGIN CERTIFICATE-----
-MIIDCTCCAfGgAwIBAgIJANegMncDqPw+MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV
-BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ
-BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL
-Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0GCSqGSIb3
-DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1osXBGO95E
-+RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgFhMrmDCES
-WDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5XiNr3brota
-aGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36ukqN6MZa
-oVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpXABuREzH9
-QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAGjHjAcMBoG
-A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAuPzT
-j4rga3RX4qN5shhgCywF+eOu3emtUohSc7QSsDmQZRL1lQ5fS/IGSler4fmxNGiD
-19deaQoWROodl1NREFGL7AqzyKM9hU30j32ztXLkntfzAb9m4UCSVGMWtrVm7TA4
-lB0ajyg0J6vJX9UWfuT1k9IZNUQKxC5qJTgd7lrIKfqW3JWCOJ42Omg0e07Z+g2y
-iKJsTwMYp+NBZzje5fb/Khzw7BoCp+hOOsMEcvhqTyimzwui2zN00RCe7LSs+LEk
-9O8OBeSdG5pA9wlmnJ2Gi3aWRujR3BDHfQtpQdynjuOjNuNCY5OMkYANJxEcLa77
-kohsawlAGjDdj6wPYg==
------END CERTIFICATE-----
diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem
deleted file mode 100644
index 3835b3eacccc0..0000000000000
--- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDNQ32YQPmwW7yu
-28ALrSaQluBiOO1osXBGO95E+RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSS
-tPgiWwU97TElZQgFhMrmDCESWDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFF
-TH3C/+LBicYSc5XiNr3brotaaGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1W
-EztbBLToxRjmLg36ukqN6MZaoVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1
-njOsqXUomnav0HpXABuREzH9QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQ
-obamHuELAgMBAAECggEBALGnokJuqiz7mTj2NSdl+6TVEOuyPbiJKpV/J4cm1XEh
-ye9qaTQcCRhH3UmcWrG75jM9KevloLRY8A1x1/lUMhtA+XJWGTU9k6a8BLut3nT4
-3X87jNTMQgSczEXNe9WudmZcxhN7rVVtOOdTpt1pP0cnCWna5HTf0D8cuLvM975j
-r1YGTjKsCF1W+tp6ZAIIMfJkUI2qBRKvSxVCSs1vZBraox3yUVnq9oRLHxZZoqOd
-d51G5phRtn6ReVPBdT8fGUBEGg3jKxTu2/vLQMUyHy0hyCAM20gzOP4FIc2g+QZU
-y42byAuc89m0OrdRWsmzHCOxcq9DwY9npaz1RscR/2ECgYEA9bHJQ0Y1afpS5gn2
-KnXenRIw9oal1utQZnohCEJ4um+K/BCEHtDnI825LPNf34IKM2rSmssvHrYN51o0
-92j9lHHXsf6MVluwsTsIu8MtNaJ1BLt96dub4ScGT6vvzObKTwsajUfIHk+FNsKq
-zps8yh1q0qyyfAcvR82+Xr6JIsMCgYEA1d+RHGewi/Ub/GCG99A1KFKsgbiIJnWB
-IFmrcyPWignhzDUcw2SV9XqAzeK8EOIHNq3e5U/tkA7aCWxtLb5UsQ8xvmwQY2cy
-X2XvSdIhO4K2PgRLgjlzZ8RHSULglqyjB2i6TjwjFl8TsRzYr6JlV6+2cMujw4Bl
-g3a8gz071BkCgYBLP7BMkmw5kRliqxph1sffg3rLhmG0eU2elTkYtoMTVqZSnRxZ
-89FW/eMBCWkLo2BMbyMhlalQ1qFbgh1GyTkhBdzx/uwsZtiu7021dAmcq6z7ThE6
-VrBfPPyJ2jcPon/DxbrUGnAIGILMSsLVlGYB4RCehZYEto6chz8O9Xw60QKBgCnd
-us1BqviqwZC04JbQJie/j09RbS2CIQXRJ9PBNzUMXCwaVYgWP5ivI1mqQcBYTqsw
-fAqNi+aAUcQ4emLS+Ec0vzsUclzTDbRJAv+DZ8f7fWtEcfeLAYFVldLMiaRVJRDF
-OnsoIII3mGY6TFyNQKNanS8VXfheQQDsFFjoera5AoGBALXYEXkESXpw4LT6qJFz
-ktQuTZDfS6LtR14/+NkYL9c5wBC4Otkg4bNbT8xGlUjethRfpkm8xRTB6zfC1/p/
-Cg6YU1cwqlkRurAhE3PEv1dCc1IDbzou8xnwqHrd6sGPDQmQ3aEtU5eJhDZKIZfx
-nQqPGK92+Jtne7+W1mFZooxs
------END PRIVATE KEY-----
diff --git a/build/regenerate_certs_for_tests.sh b/build/regenerate_certs_for_tests.sh
index fff1c057060f3..9582a7496cd1d 100755
--- a/build/regenerate_certs_for_tests.sh
+++ b/build/regenerate_certs_for_tests.sh
@@ -68,13 +68,6 @@ reissue_certificate_no_subject \
   $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem \
   $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem
 
-generate_ca
-cp ca-cert.pem $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem
-reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem \
-  $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem
-reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-key.pem \
-  $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem
-
 generate_ca
 cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem
 reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-key.pem \
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java
index 19a550457a4dd..0e4f1bccc814d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BrokerAdminClientTlsAuthTest.java
@@ -63,7 +63,7 @@ private void buildConf(ServiceConfiguration conf) {
         conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setAuthenticationEnabled(true);
-        conf.setSuperUserRoles(Set.of("superproxy", "broker.pulsar.apache.org"));
+        conf.setSuperUserRoles(Set.of("superproxy", "broker-localhost-SAN"));
         conf.setAuthenticationProviders(
                 Set.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls"));
         conf.setAuthorizationEnabled(true);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java
index e170425ffe443..d3d4b7cf9341c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java
@@ -323,6 +323,9 @@ public Builder spyConfigCustomizer(Consumer spyConfigCustomiz
          */
         public Builder configCustomizer(Consumer configCustomerizer) {
             configCustomerizer.accept(svcConfig);
+            if (config != null) {
+                configCustomerizer.accept(config);
+            }
             return this;
         }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java
index 71e30c21b3c20..75ae91f18a305 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java
@@ -64,12 +64,6 @@
 public class AuthenticatedProducerConsumerTest extends ProducerConsumerBase {
     private static final Logger log = LoggerFactory.getLogger(AuthenticatedProducerConsumerTest.class);
 
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-
     private final String BASIC_CONF_FILE_PATH = "./src/test/resources/authentication/basic/.htpasswd";
 
     private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256);
@@ -88,9 +82,9 @@ protected void setup() throws Exception {
 
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(true);
         conf.setTopicLevelPoliciesEnabled(false);
 
@@ -104,7 +98,8 @@ protected void setup() throws Exception {
         conf.setBrokerClientTlsEnabled(true);
         conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
         conf.setBrokerClientAuthenticationParameters(
-                "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH);
+                "tlsCertFile:" + getTlsFileForClient("admin.cert")
+                        + ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8"));
 
         Set providers = new HashSet<>();
         providers.add(AuthenticationProviderTls.class.getName());
@@ -126,7 +121,7 @@ protected void setup() throws Exception {
 
     protected final void internalSetup(Authentication auth) throws Exception {
         admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth)
                 .build());
         String lookupUrl;
         // For http basic authentication test
@@ -136,7 +131,7 @@ protected final void internalSetup(Authentication auth) throws Exception {
             lookupUrl = pulsar.getBrokerServiceUrlTls();
         }
         replacePulsarClient(PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth)
                 .enableTls(true));
     }
 
@@ -188,8 +183,8 @@ public void testTlsSyncProducerAndConsumer(int batchMessageDelayMs) throws Excep
         log.info("-- Starting {} test --", methodName);
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         internalSetup(authTls);
@@ -246,8 +241,8 @@ public void testAnonymousSyncProducerAndConsumer(int batchMessageDelayMs) throws
         log.info("-- Starting {} test --", methodName);
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         internalSetup(authTls);
@@ -291,8 +286,8 @@ public void testAuthenticationFilterNegative() throws Exception {
         log.info("-- Starting {} test --", methodName);
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         internalSetup(authTls);
@@ -324,8 +319,8 @@ public void testInternalServerExceptionOnLookup() throws Exception {
         log.info("-- Starting {} test --", methodName);
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         internalSetup(authTls);
@@ -362,8 +357,8 @@ public void testInternalServerExceptionOnLookup() throws Exception {
     @Test
     public void testDeleteAuthenticationPoliciesOfTopic() throws Exception {
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         internalSetup(authTls);
@@ -424,7 +419,8 @@ public void testDeleteAuthenticationPoliciesOfTopic() throws Exception {
         admin.clusters().deleteCluster("test");
     }
 
-    private final Authentication tlsAuth = new AuthenticationTls(TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH);
+    private final Authentication tlsAuth =
+            new AuthenticationTls(getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"));
     private final Authentication tokenAuth = new AuthenticationToken(ADMIN_TOKEN);
 
     @DataProvider
@@ -454,10 +450,9 @@ public void testTlsTransportWithAnyAuth(Supplier url, Authentication aut
 
         @Cleanup
         PulsarClient client = PulsarClient.builder().serviceUrl(url.get())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
-                .tlsKeyFilePath(TLS_CLIENT_KEY_FILE_PATH)
-                .tlsCertificateFilePath(TLS_CLIENT_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
+                .tlsKeyFilePath(getTlsFileForClient("admin.key-pk8"))
+                .tlsCertificateFilePath(getTlsFileForClient("admin.cert"))
                 .authentication(auth)
                 .allowTlsInsecureConnection(false)
                 .enableTlsHostnameVerification(false)
@@ -470,8 +465,8 @@ public void testTlsTransportWithAnyAuth(Supplier url, Authentication aut
     @Test
     public void testCleanupEmptyTopicAuthenticationMap() throws Exception {
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         internalSetup(authTls);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
index f2631f591217b..e3bd321d76332 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
@@ -46,17 +46,10 @@ public class AuthenticationTlsHostnameVerificationTest extends ProducerConsumerB
     private final String TLS_MIM_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/hn-verification/broker-cert.pem";
     private final String TLS_MIM_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/hn-verification/broker-key.pem";
 
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-
     private final String BASIC_CONF_FILE_PATH = "./src/test/resources/authentication/basic/.htpasswd";
 
     private boolean hostnameVerificationEnabled = true;
-    private String clientTrustCertFilePath = TLS_TRUST_CERT_FILE_PATH;
+    private String clientTrustCertFilePath = CA_CERT_FILE_PATH;
 
     protected void setup() throws Exception {
         super.internalSetup();
@@ -81,7 +74,8 @@ protected void setup() throws Exception {
 
         conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
         conf.setBrokerClientAuthenticationParameters(
-                "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SERVER_KEY_FILE_PATH);
+                "tlsCertFile:" + getTlsFileForClient("admin.cert")
+                        + ",tlsKeyFile:" +  getTlsFileForClient("admin.key-pk8"));
 
         Set providers = new HashSet<>();
         providers.add(AuthenticationProviderTls.class.getName());
@@ -100,8 +94,8 @@ protected void setup() throws Exception {
     protected void setupClient() throws Exception {
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
 
@@ -147,11 +141,11 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname
         // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost"
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH);
         conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH);
         conf.setBrokerClientAuthenticationParameters(
-                "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_MIM_SERVER_KEY_FILE_PATH);
+                "tlsCertFile:" + getTlsFileForClient("admin.cert") + "," + "tlsKeyFile:" + TLS_MIM_SERVER_KEY_FILE_PATH);
 
         setup();
 
@@ -188,9 +182,9 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception {
         // setup broker cert which has CN = "localhost"
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
 
         setup();
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java
index 186bf9d736e45..c9b243257c4e1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java
@@ -37,15 +37,9 @@
 
 @Test(groups = "broker-api")
 public class ClientAuthenticationTlsTest extends ProducerConsumerBase {
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
 
     private final Authentication authenticationTls =
-            new AuthenticationTls(TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH);
+            new AuthenticationTls(getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"));
 
     @Override
     protected void doInitConf() throws Exception {
@@ -57,17 +51,18 @@ protected void doInitConf() throws Exception {
         providers.add(AuthenticationProviderTls.class.getName());
         conf.setAuthenticationProviders(providers);
 
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
 
         conf.setTlsAllowInsecureConnection(false);
 
         conf.setBrokerClientTlsEnabled(true);
         conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
         conf.setBrokerClientAuthenticationParameters(
-                "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH);
-        conf.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+                "tlsCertFile:" + getTlsFileForClient("admin.cert")
+                        + ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8"));
+        conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
     }
 
     @BeforeClass(alwaysRun = true)
@@ -94,7 +89,7 @@ public void testAdminWithTrustCert() throws PulsarClientException, PulsarAdminEx
         @Cleanup
         PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(getPulsar().getWebServiceAddressTls())
                 .sslProvider("JDK")
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .build();
         pulsarAdmin.clusters().getClusters();
     }
@@ -105,7 +100,7 @@ public void testAdminWithFull() throws PulsarClientException, PulsarAdminExcepti
         PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(getPulsar().getWebServiceAddressTls())
                 .sslProvider("JDK")
                 .authentication(authenticationTls)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .build();
         pulsarAdmin.clusters().getClusters();
     }
@@ -139,7 +134,7 @@ public void testClientWithTrustCert() throws PulsarClientException, PulsarAdminE
         PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(getPulsar().getBrokerServiceUrlTls())
                 .sslProvider("JDK")
                 .operationTimeout(3, TimeUnit.SECONDS)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .build();
         @Cleanup
         Producer ignored = pulsarClient.newProducer().topic(UUID.randomUUID().toString()).create();
@@ -152,7 +147,7 @@ public void testClientWithFull() throws PulsarClientException, PulsarAdminExcept
                 .sslProvider("JDK")
                 .operationTimeout(3, TimeUnit.SECONDS)
                 .authentication(authenticationTls)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .build();
         @Cleanup
         Producer ignored = pulsarClient.newProducer().topic(UUID.randomUUID().toString()).create();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java
index ca58bddf13c23..f58c1fa26afc7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java
@@ -31,11 +31,6 @@
 import org.testng.annotations.BeforeMethod;
 
 public abstract class ProducerConsumerBase extends MockedPulsarServiceBaseTest {
-    protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
 
     protected String methodName;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java
index 7f632d5a76485..19009689dc8a1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProxyProtocolTest.java
@@ -45,11 +45,11 @@ public void testSniProxyProtocol() throws Exception {
         String topicName = "persistent://my-property/use/my-ns/my-topic1";
 
         ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(brokerServiceUrl)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
                 .proxyServiceUrl(proxyUrl, ProxyProtocol.SNI).operationTimeout(1000, TimeUnit.MILLISECONDS);
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         clientBuilder.authentication(AuthenticationTls.class.getName(), authParams);
 
         @Cleanup
@@ -68,11 +68,11 @@ public void testSniProxyProtocolWithInvalidProxyUrl() throws Exception {
         String topicName = "persistent://my-property/use/my-ns/my-topic1";
 
         ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(brokerServiceUrl)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
                 .proxyServiceUrl(proxyUrl, ProxyProtocol.SNI).operationTimeout(1000, TimeUnit.MILLISECONDS);
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         clientBuilder.authentication(AuthenticationTls.class.getName(), authParams);
 
         @Cleanup
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java
index 95a78d7ffcef6..fff61c5c8c940 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.impl.auth.AuthenticationTls;
@@ -30,21 +31,38 @@
 @Test(groups = "broker-api")
 public class TlsHostVerificationTest extends TlsProducerConsumerBase {
 
+    @Override
+    @Test(enabled = false)
+    protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) {
+        builder.configCustomizer(config -> {
+            // Advertise a hostname that routes but is not on the certificate
+            // Note that if you are on a Mac, you'll need to run the following to make loopback work for 127.0.0.2
+            // $ sudo ifconfig lo0 alias 127.0.0.2 up
+            config.setAdvertisedAddress("127.0.0.2");
+        });
+    }
+
     @Test
     public void testTlsHostVerificationAdminClient() throws Exception {
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
-        String websocketTlsAddress = pulsar.getWebServiceAddressTls();
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
+        Assert.assertTrue(pulsar.getWebServiceAddressTls().startsWith("https://127.0.0.2:"),
+                "Test relies on this address");
         PulsarAdmin adminClientTls = PulsarAdmin.builder()
-                .serviceHttpUrl(websocketTlsAddress.replace("localhost", "127.0.0.1"))
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false)
+                .serviceHttpUrl(pulsar.getWebServiceAddressTls())
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false)
                 .authentication(AuthenticationTls.class.getName(), authParams).enableTlsHostnameVerification(true)
+                .requestTimeout(1, java.util.concurrent.TimeUnit.SECONDS)
                 .build();
 
         try {
             adminClientTls.tenants().getTenants();
             Assert.fail("Admin call should be failed due to hostnameVerification enabled");
+        } catch (PulsarAdminException.TimeoutException e) {
+            // The test was previously able to fail here, but that is not the right way for the test to pass.
+            // If you hit this error and are running on OSX, you may need to run "sudo ifconfig lo0 alias 127.0.0.2 up"
+            Assert.fail("Admin call should not timeout, it should fail due to SSL error");
         } catch (PulsarAdminException e) {
             // Ok
         }
@@ -53,11 +71,13 @@ public void testTlsHostVerificationAdminClient() throws Exception {
     @Test
     public void testTlsHostVerificationDisabledAdminClient() throws Exception {
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
+        Assert.assertTrue(pulsar.getWebServiceAddressTls().startsWith("https://127.0.0.2:"),
+                "Test relies on this address");
         PulsarAdmin adminClient = PulsarAdmin.builder()
                 .serviceHttpUrl(pulsar.getWebServiceAddressTls())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false)
                 .authentication(AuthenticationTls.class.getName(), authParams).enableTlsHostnameVerification(false)
                 .build();
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java
index 6a2109836a2c9..39bab20d97df5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java
@@ -38,11 +38,6 @@
 
 @Test(groups = "broker-api")
 public abstract class TlsProducerConsumerBase extends ProducerConsumerBase {
-    protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
     private final String clusterName = "use";
 
     @BeforeMethod
@@ -64,9 +59,9 @@ protected void cleanup() throws Exception {
     protected void internalSetUpForBroker() {
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setClusterName(clusterName);
         conf.setTlsRequireTrustedClientCertOnConnect(true);
         Set tlsProtocols = Sets.newConcurrentHashSet();
@@ -81,12 +76,12 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl)
             pulsarClient.close();
         }
         ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(lookupUrl)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).enableTls(true).allowTlsInsecureConnection(false)
                 .operationTimeout(1000, TimeUnit.MILLISECONDS);
         if (addCertificates) {
             Map authParams = new HashMap<>();
-            authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-            authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+            authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+            authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
             clientBuilder.authentication(AuthenticationTls.class.getName(), authParams);
         }
         replacePulsarClient(clientBuilder);
@@ -94,15 +89,15 @@ protected void internalSetUpForClient(boolean addCertificates, String lookupUrl)
 
     protected void internalSetUpForNamespace() throws Exception {
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         if (admin != null) {
             admin.close();
         }
 
         admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false)
                 .authentication(AuthenticationTls.class.getName(), authParams).build());
         admin.clusters().createCluster(clusterName,
                 ClusterData.builder()
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java
index 0563fc3b9da37..879289eb65dc8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java
@@ -146,9 +146,9 @@ public void testTlsCertsFromDynamicStream() throws Exception {
                 .operationTimeout(1000, TimeUnit.MILLISECONDS);
         AtomicInteger index = new AtomicInteger(0);
 
-        ByteArrayInputStream certStream = createByteInputStream(TLS_CLIENT_CERT_FILE_PATH);
-        ByteArrayInputStream keyStream = createByteInputStream(TLS_CLIENT_KEY_FILE_PATH);
-        ByteArrayInputStream trustStoreStream = createByteInputStream(TLS_TRUST_CERT_FILE_PATH);
+        ByteArrayInputStream certStream = createByteInputStream(getTlsFileForClient("admin.cert"));
+        ByteArrayInputStream keyStream = createByteInputStream(getTlsFileForClient("admin.key-pk8"));
+        ByteArrayInputStream trustStoreStream = createByteInputStream(CA_CERT_FILE_PATH);
 
         Supplier certProvider = () -> getStream(index, certStream);
         Supplier keyProvider = () -> getStream(index, keyStream);
@@ -203,9 +203,9 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception
         AtomicInteger certIndex = new AtomicInteger(1);
         AtomicInteger keyIndex = new AtomicInteger(0);
         AtomicInteger trustStoreIndex = new AtomicInteger(1);
-        ByteArrayInputStream certStream = createByteInputStream(TLS_CLIENT_CERT_FILE_PATH);
-        ByteArrayInputStream keyStream = createByteInputStream(TLS_CLIENT_KEY_FILE_PATH);
-        ByteArrayInputStream trustStoreStream = createByteInputStream(TLS_TRUST_CERT_FILE_PATH);
+        ByteArrayInputStream certStream = createByteInputStream(getTlsFileForClient("admin.cert"));
+        ByteArrayInputStream keyStream = createByteInputStream(getTlsFileForClient("admin.key-pk8"));
+        ByteArrayInputStream trustStoreStream = createByteInputStream(CA_CERT_FILE_PATH);
         Supplier certProvider = () -> getStream(certIndex, certStream,
                 keyStream/* invalid cert file */);
         Supplier keyProvider = () -> getStream(keyIndex, keyStream);
@@ -252,7 +252,8 @@ private ByteArrayInputStream getStream(AtomicInteger index, ByteArrayInputStream
         return streams[index.intValue()];
     }
 
-    private final Authentication tlsAuth = new AuthenticationTls(TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH);
+    private final Authentication tlsAuth =
+        new AuthenticationTls(getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"));
 
     @DataProvider
     public Object[] tlsTransport() {
@@ -276,13 +277,14 @@ public void testTlsTransport(Supplier url, Authentication auth) throws E
         internalSetUpForNamespace();
 
         ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(url.get())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .allowTlsInsecureConnection(false)
                 .enableTlsHostnameVerification(false)
                 .authentication(auth);
 
         if (auth == null) {
-            clientBuilder.tlsKeyFilePath(TLS_CLIENT_KEY_FILE_PATH).tlsCertificateFilePath(TLS_CLIENT_CERT_FILE_PATH);
+            clientBuilder.tlsKeyFilePath(getTlsFileForClient("admin.key-pk8"))
+                    .tlsCertificateFilePath(getTlsFileForClient("admin.cert"));
         }
 
         @Cleanup
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java
index fd722e52e5f1f..173fa8acb0fdd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsSniTest.java
@@ -50,12 +50,12 @@ public void testIpAddressInBrokerServiceUrl() throws Exception {
                     brokerServiceUrlTls.getPort());
 
         ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(brokerServiceIpAddressUrl)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(false)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false)
                 .enableTlsHostnameVerification(false)
                 .operationTimeout(1000, TimeUnit.MILLISECONDS);
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         clientBuilder.authentication(AuthenticationTls.class.getName(), authParams);
 
         @Cleanup
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java
index e955a9ae7062f..4fc0d315d2253 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java
@@ -101,9 +101,9 @@ public String getExpireToken(String role, Date date) {
     protected void internalSetUpForBroker() {
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setClusterName(configClusterName);
         conf.setAuthenticationRefreshCheckSeconds(1);
         conf.setTlsRequireTrustedClientCertOnConnect(false);
@@ -121,7 +121,7 @@ protected void internalSetUpForBroker() {
     private PulsarClient getClient(String token) throws Exception {
         ClientBuilder clientBuilder = PulsarClient.builder()
                 .serviceUrl(pulsar.getBrokerServiceUrlTls())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .enableTls(true)
                 .allowTlsInsecureConnection(false)
                 .enableTlsHostnameVerification(true)
@@ -132,7 +132,7 @@ private PulsarClient getClient(String token) throws Exception {
 
     private PulsarAdmin getAdmin(String token) throws Exception {
         PulsarAdminBuilder clientBuilder = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddressTls())
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .allowTlsInsecureConnection(false)
                 .authentication(AuthenticationToken.class.getName(),"token:" +token)
                 .enableTlsHostnameVerification(true);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java
index c832cba163d63..aa190cd2e0a73 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java
@@ -89,6 +89,7 @@
 import org.apache.pulsar.functions.utils.FunctionCommon;
 import org.apache.pulsar.io.core.Sink;
 import org.apache.pulsar.io.core.SinkContext;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -121,11 +122,16 @@ public class PulsarFunctionLocalRunTest {
 
     private static final String CLUSTER = "local";
 
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
+    private final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    private final String TLS_TRUST_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
 
     private static final String SYSTEM_PROPERTY_NAME_NAR_FILE_PATH = "pulsar-io-data-generator.nar.path";
     private PulsarFunctionTestTemporaryDirectory tempDirectory;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java
index 95923586fe23e..c820f512a68de 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java
@@ -71,6 +71,7 @@
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -99,11 +100,16 @@ public class PulsarFunctionPublishTest {
     String primaryHost;
     String workerId;
 
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
+    private final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    private final String TLS_TRUST_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
     private PulsarFunctionTestTemporaryDirectory tempDirectory;
 
     @DataProvider(name = "validRoleName")
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java
index 9991e9f1b70fd..3a99cc647ed5c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java
@@ -62,6 +62,7 @@
 import org.apache.pulsar.functions.worker.PulsarWorkerService;
 import org.apache.pulsar.functions.worker.WorkerConfig;
 import org.apache.pulsar.functions.worker.WorkerService;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.awaitility.Awaitility;
 import org.slf4j.Logger;
@@ -75,11 +76,16 @@ public abstract class AbstractPulsarE2ETest {
 
 	public static final Logger log = LoggerFactory.getLogger(AbstractPulsarE2ETest.class);
 
-    protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    protected final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    protected final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
+    protected final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    protected final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    protected final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    protected final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    protected final String TLS_TRUST_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
     protected final String tenant = "external-repl-prop";
 
 	protected LocalBookkeeperEnsemble bkEnsemble;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java
index 16218f6ce6448..d31d0c66bdf93 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java
@@ -51,6 +51,7 @@
 import org.apache.pulsar.functions.worker.PulsarWorkerService;
 import org.apache.pulsar.functions.worker.WorkerConfig;
 import org.apache.pulsar.functions.worker.WorkerService;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -77,10 +78,16 @@ public class PulsarFunctionAdminTest {
     String pulsarFunctionsNamespace = tenant + "/pulsar-function-admin";
     String primaryHost;
 
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
+    private final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    private final String TLS_TRUST_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
 
     private static final Logger log = LoggerFactory.getLogger(PulsarFunctionAdminTest.class);
 
@@ -113,8 +120,7 @@ void setup(Method method) throws Exception {
         config.setAuthenticationProviders(providers);
         config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
         config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        config.setTlsAllowInsecureConnection(true);
-
+        config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
 
         functionsWorkerService = createPulsarFunctionWorker(config);
         Optional functionWorkerService = Optional.of(functionsWorkerService);
@@ -132,7 +138,6 @@ void setup(Method method) throws Exception {
                 PulsarAdmin.builder()
                         .serviceHttpUrl(pulsar.getWebServiceAddressTls())
                         .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH)
-                        .allowTlsInsecureConnection(true)
                         .authentication(authTls)
                         .build());
 
@@ -203,7 +208,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf
         workerConfig.setBrokerClientAuthenticationParameters(
                 String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH));
         workerConfig.setUseTls(true);
-        workerConfig.setTlsAllowInsecureConnection(true);
         workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
 
         PulsarWorkerService workerService = new PulsarWorkerService();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java
index 5de3d4f7e0868..810ac69ac3eb3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java
@@ -66,6 +66,7 @@
 import org.apache.pulsar.functions.worker.PulsarWorkerService.PulsarClientCreator;
 import org.apache.pulsar.functions.worker.WorkerConfig;
 import org.apache.pulsar.functions.worker.rest.WorkerServer;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -90,10 +91,16 @@ public class PulsarFunctionTlsTest {
     PulsarAdmin functionAdmin;
     private final List namespaceList = new LinkedList<>();
 
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
+    private final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    private final String TLS_TRUST_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
 
     private static final Logger log = LoggerFactory.getLogger(PulsarFunctionTlsTest.class);
     private PulsarFunctionTestTemporaryDirectory tempDirectory;
@@ -121,7 +128,7 @@ void setup(Method method) throws Exception {
         config.setAuthenticationProviders(providers);
         config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
         config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        config.setTlsAllowInsecureConnection(true);
+        config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
         config.setAdvertisedAddress("localhost");
 
         PulsarAdmin admin = mock(PulsarAdmin.class);
@@ -163,7 +170,7 @@ void setup(Method method) throws Exception {
         authTls.configure(authParams);
 
         functionAdmin = PulsarAdmin.builder().serviceHttpUrl(functionTlsUrl)
-                .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH).allowTlsInsecureConnection(true)
+                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
                 .authentication(authTls).build();
 
         Thread.sleep(100);
@@ -217,7 +224,7 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf
                 String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH));
         workerConfig.setUseTls(true);
         workerConfig.setTlsAllowInsecureConnection(true);
-        workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
+        workerConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
 
         workerConfig.setWorkerPortTls(0);
         workerConfig.setTlsEnabled(true);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java
index 3ee9b6127de7b..91cd4fab470d6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java
@@ -64,12 +64,13 @@ public void setup() throws Exception {
         config.setWebServicePort(Optional.of(0));
         config.setWebServicePortTls(Optional.of(0));
         config.setBrokerClientTlsEnabled(true);
-        config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
-        config.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
+        config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        config.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         config.setClusterName("use");
-        config.setBrokerClientAuthenticationParameters("tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + ",tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH);
+        config.setBrokerClientAuthenticationParameters("tlsCertFile:" + getTlsFileForClient("admin.cert") +
+                ",tlsKeyFile:" + getTlsFileForClient("admin.key-pk8"));
         config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
         config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE);
         service = spyWithClassAndConstructorArgs(WebSocketService.class, config);
@@ -103,7 +104,7 @@ public void socketTest() throws GeneralSecurityException {
 
         SslContextFactory sslContextFactory = new SslContextFactory();
         sslContextFactory.setSslContext(SecurityUtility
-                .createSslContext(false, SecurityUtility.loadCertificatesFromPemFile(TLS_TRUST_CERT_FILE_PATH), null));
+                .createSslContext(false, SecurityUtility.loadCertificatesFromPemFile(CA_CERT_FILE_PATH), null));
 
         WebSocketClient consumeClient = new WebSocketClient(sslContextFactory);
         SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket();
diff --git a/tests/certificate-authority/.gitignore b/tests/certificate-authority/.gitignore
new file mode 100644
index 0000000000000..de3be7546361f
--- /dev/null
+++ b/tests/certificate-authority/.gitignore
@@ -0,0 +1,3 @@
+# Files generated when running openssl
+*.old
+*.attr
diff --git a/tests/certificate-authority/README.md b/tests/certificate-authority/README.md
index 008120a35f4a1..02ebbdf9258d5 100644
--- a/tests/certificate-authority/README.md
+++ b/tests/certificate-authority/README.md
@@ -3,23 +3,33 @@
 Generated based on instructions from https://jamielinux.com/docs/openssl-certificate-authority/introduction.html,
 though the intermediate CA has been omitted for simplicity.
 
-The environment variable, CA_HOME, must be set to point to the directory
-containing this file before running any openssl commands.
+The following commands must be run in the same directory as this README due to the configuration for the openssl.cnf file.
 
 The password for the CA private key is ```PulsarTesting```.
 
 ## Generating server keys
 
-In this example, we're generating a key for the broker.
+In this example, we're generating a key for the broker and the proxy. If there is a need to create them again, a new
+CN will need to be used because we have the index.txt database in this directory. It's also possible that we could
+remove this file and start over. At the time of adding this change, I didn't see a need to change the paradigm.
 
-The common name when generating the CSR should be the domain name of the broker.
+The common name when generating the CSR used to be the domain name of the broker. However, now we rely on the Subject
+Alternative Name, or the SAN, to be the domain name. This is because the CN is deprecated in the certificate spec. The
+[openssl.cnf](openssl.cnf) file has been updated to reflect this change. The proxy and the broker have the following
+SAN: ```DNS:localhost, IP:127.0.0.1```.
 
 ```bash
 openssl genrsa -out server-keys/broker.key.pem 2048
-openssl req -config openssl.cnf -key server-keys/broker.key.pem -new -sha256 -out server-keys/broker.csr.pem
-openssl ca -config openssl.cnf -extensions server_cert \
-    -days 100000 -notext -md sha256 -in server-keys/broker.csr.pem -out server-keys/broker.cert.pem
+openssl req -config openssl.cnf -subj "/CN=broker-localhost-SAN" -key server-keys/broker.key.pem -new -sha256 -out server-keys/broker.csr.pem
+openssl ca -config openssl.cnf -extensions broker_cert -days 100000 -md sha256 -in server-keys/broker.csr.pem \
+    -out server-keys/broker.cert.pem -batch -key PulsarTesting
 openssl pkcs8 -topk8 -inform PEM -outform PEM -in server-keys/broker.key.pem -out server-keys/broker.key-pk8.pem -nocrypt
+
+openssl genrsa -out server-keys/proxy.key.pem 2048
+openssl req -config openssl.cnf -subj "/CN=proxy-localhost-SAN" -key server-keys/proxy.key.pem -new -sha256 -out server-keys/proxy.csr.pem
+openssl ca -config openssl.cnf -extensions proxy_cert -days 100000 -md sha256 -in server-keys/proxy.csr.pem \
+    -out server-keys/proxy.cert.pem -batch -key PulsarTesting
+openssl pkcs8 -topk8 -inform PEM -outform PEM -in server-keys/proxy.key.pem -out server-keys/proxy.key-pk8.pem -nocrypt
 ```
 
 You need to configure the server with broker.key-pk8.pem and broker.cert.pem.
diff --git a/tests/certificate-authority/index.txt b/tests/certificate-authority/index.txt
index 376f86725c21a..acb5eed051c13 100644
--- a/tests/certificate-authority/index.txt
+++ b/tests/certificate-authority/index.txt
@@ -5,3 +5,5 @@ V	22920409135604Z		1003	unknown	/CN=proxy
 V	22920410132517Z		1004	unknown	/CN=superproxy
 V	22920411084025Z		1005	unknown	/CN=user1
 V	22960802101401Z		1006	unknown	/CN=proxy.pulsar.apache.org
+V	22970222155018Z		1007	unknown	/CN=broker-localhost-SAN
+V	22970222155019Z		1008	unknown	/CN=proxy-localhost-SAN
diff --git a/tests/certificate-authority/newcerts/1007.pem b/tests/certificate-authority/newcerts/1007.pem
new file mode 100644
index 0000000000000..4237719f20ebd
--- /dev/null
+++ b/tests/certificate-authority/newcerts/1007.pem
@@ -0,0 +1,111 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 4103 (0x1007)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: CN=foobar
+        Validity
+            Not Before: May 10 15:50:18 2023 GMT
+            Not After : Feb 22 15:50:18 2297 GMT
+        Subject: CN=broker-localhost-SAN
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:de:d1:da:bb:91:b3:16:c4:b2:e8:89:30:9e:c1:
+                    5e:0b:cf:db:c4:c3:d9:b1:af:40:a5:0b:38:36:1b:
+                    14:fe:0f:22:9c:e6:59:6a:15:5b:db:f6:f7:f3:a5:
+                    02:29:94:7a:d2:0c:67:ad:aa:63:62:7e:fc:58:11:
+                    29:48:b8:3c:91:b2:73:7e:12:6b:f2:ea:36:77:0f:
+                    15:9b:46:95:ce:73:15:8d:c8:d9:97:57:03:90:33:
+                    2d:7d:f3:ee:e5:01:6d:d8:c6:da:ab:07:b9:dd:1c:
+                    e0:4b:ce:6a:de:a8:d2:e3:c1:52:6d:83:3a:0a:f0:
+                    ed:cf:f7:56:6a:87:0e:73:e3:12:82:2b:65:ab:d8:
+                    a9:44:5b:4a:2f:a5:92:94:32:f1:a1:e4:af:18:0f:
+                    0f:18:60:cd:f7:d0:9d:03:9f:d7:e9:a8:60:54:bb:
+                    3b:9a:05:db:fd:38:04:3c:b4:23:41:16:6c:7c:3b:
+                    d9:b6:e0:2f:bd:cb:62:55:1b:e8:d0:8f:43:76:ef:
+                    55:86:cf:25:c3:bc:ae:e3:46:50:89:f7:71:ad:06:
+                    5e:28:e6:f6:f0:76:27:ea:7e:1b:67:53:39:26:20:
+                    19:18:82:b1:11:5f:ea:91:c2:e3:d3:f6:5a:c7:fd:
+                    61:a2:92:de:7d:7c:da:6d:e8:bf:39:52:10:31:60:
+                    4b:e1
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Cert Type: 
+                SSL Server
+            Netscape Comment: 
+                OpenSSL Generated Server Certificate
+            X509v3 Subject Key Identifier: 
+                17:07:3B:AA:85:83:B5:04:83:EC:B2:6C:1E:3A:F0:F5:59:AA:61:28
+            X509v3 Subject Alternative Name: 
+                DNS:localhost, DNS:unresolvable-broker-address, IP Address:127.0.0.1
+            X509v3 Authority Key Identifier: 
+                keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27
+                DirName:/CN=foobar
+                serial:D7:E2:87:4F:A0:79:E2:0C
+
+            X509v3 Key Usage: critical
+                Digital Signature, Key Encipherment
+            X509v3 Extended Key Usage: 
+                TLS Web Server Authentication
+    Signature Algorithm: sha256WithRSAEncryption
+         e4:27:61:e2:0f:b6:a0:ca:9f:ce:e3:53:0b:44:ab:86:a1:e2:
+         4d:88:e1:7d:2e:b0:aa:32:96:2b:3d:da:60:70:6a:c3:62:c5:
+         76:f2:8f:0d:16:31:f2:ad:e5:2f:43:f3:cb:e4:fa:95:6c:20:
+         81:33:1a:c7:5a:55:57:c9:ab:ca:66:45:30:58:00:db:e8:51:
+         c9:2c:a9:72:c1:18:f5:01:87:9f:73:20:85:6c:e5:6c:3f:c9:
+         67:b4:f0:20:e5:ed:e2:4a:08:0b:af:68:43:e5:a9:c7:e1:39:
+         e8:b5:49:cb:47:4a:6d:e5:16:ae:88:92:13:85:8e:42:1e:0a:
+         eb:59:ed:a7:c1:9b:bc:4b:7b:99:f8:1d:f0:d7:1d:90:c9:cf:
+         86:6a:d3:10:d0:36:e4:f5:b9:33:79:c7:a2:68:31:f7:bb:8d:
+         1e:d6:33:79:bd:e7:0e:4f:4d:e9:2e:15:04:4f:6b:4b:2e:93:
+         28:72:d1:0e:aa:ee:e6:ef:68:be:58:2b:cc:56:01:27:16:f9:
+         34:8e:66:86:27:0a:b0:fb:32:56:a9:8a:d9:6f:b1:86:bd:ba:
+         fd:50:6c:d5:b2:54:e7:4e:c6:2d:19:88:a9:89:2c:ef:be:08:
+         0d:2b:49:91:0b:09:42:64:06:a3:9d:d7:94:ed:e8:74:74:48:
+         43:57:41:6f:e5:06:98:46:1d:c5:60:9c:69:f8:fb:fe:a6:01:
+         4a:35:be:21:36:c2:a3:44:c8:c4:2c:21:09:f4:28:9a:ad:a0:
+         97:1e:00:29:cc:0f:26:fa:59:21:25:c0:9e:fa:22:53:67:6d:
+         ab:a6:56:08:fd:37:1d:69:fe:ef:6f:29:89:1a:66:7b:c7:ff:
+         b1:34:f1:d6:be:21:81:e3:bc:4f:13:02:a7:4b:9d:13:05:46:
+         40:88:4a:aa:db:fb:64:f8:6b:fb:5d:a0:b1:0c:1a:b8:4c:ab:
+         6f:69:fe:0b:55:4e:b3:38:1f:91:0b:71:77:1e:11:39:54:9a:
+         62:51:ea:6d:a8:5e:0d:4a:91:fb:d8:be:5d:93:e8:43:f3:4a:
+         11:fb:31:cf:14:1a:1c:8d:31:1b:99:31:e0:2b:81:01:91:6f:
+         da:ba:cb:1f:51:21:55:29:3f:4c:71:e3:d0:29:41:de:a0:00:
+         da:07:ed:5e:c9:af:32:61:6d:55:f8:f5:2d:46:03:34:33:fb:
+         2e:1e:aa:7c:fe:d2:30:4d:40:cc:ed:76:ec:f6:bd:ed:35:c8:
+         d8:b3:46:56:aa:2c:53:84:56:45:b0:a3:f6:35:66:93:da:8c:
+         17:39:c1:29:7c:99:c5:0b:73:c1:f9:16:d0:57:fc:57:59:06:
+         af:39:9f:a9:51:35:0b:c7
+-----BEGIN CERTIFICATE-----
+MIIExzCCAq+gAwIBAgICEAcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
+YmFyMCAXDTIzMDUxMDE1NTAxOFoYDzIyOTcwMjIyMTU1MDE4WjAfMR0wGwYDVQQD
+DBRicm9rZXItbG9jYWxob3N0LVNBTjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAN7R2ruRsxbEsuiJMJ7BXgvP28TD2bGvQKULODYbFP4PIpzmWWoVW9v2
+9/OlAimUetIMZ62qY2J+/FgRKUi4PJGyc34Sa/LqNncPFZtGlc5zFY3I2ZdXA5Az
+LX3z7uUBbdjG2qsHud0c4EvOat6o0uPBUm2DOgrw7c/3VmqHDnPjEoIrZavYqURb
+Si+lkpQy8aHkrxgPDxhgzffQnQOf1+moYFS7O5oF2/04BDy0I0EWbHw72bbgL73L
+YlUb6NCPQ3bvVYbPJcO8ruNGUIn3ca0GXijm9vB2J+p+G2dTOSYgGRiCsRFf6pHC
+49P2Wsf9YaKS3n182m3ovzlSEDFgS+ECAwEAAaOCARcwggETMAkGA1UdEwQCMAAw
+EQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVy
+YXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBcHO6qFg7UEg+yybB46
+8PVZqmEoMDcGA1UdEQQwMC6CCWxvY2FsaG9zdIIbdW5yZXNvbHZhYmxlLWJyb2tl
+ci1hZGRyZXNzhwR/AAABMEEGA1UdIwQ6MDiAFFcL6csj6L9HPlB6P0V+oRhDnRUn
+oRWkEzARMQ8wDQYDVQQDDAZmb29iYXKCCQDX4odPoHniDDAOBgNVHQ8BAf8EBAMC
+BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAOQnYeIP
+tqDKn87jUwtEq4ah4k2I4X0usKoylis92mBwasNixXbyjw0WMfKt5S9D88vk+pVs
+IIEzGsdaVVfJq8pmRTBYANvoUcksqXLBGPUBh59zIIVs5Ww/yWe08CDl7eJKCAuv
+aEPlqcfhOei1SctHSm3lFq6IkhOFjkIeCutZ7afBm7xLe5n4HfDXHZDJz4Zq0xDQ
+NuT1uTN5x6JoMfe7jR7WM3m95w5PTekuFQRPa0sukyhy0Q6q7ubvaL5YK8xWAScW
++TSOZoYnCrD7MlapitlvsYa9uv1QbNWyVOdOxi0ZiKmJLO++CA0rSZELCUJkBqOd
+15Tt6HR0SENXQW/lBphGHcVgnGn4+/6mAUo1viE2wqNEyMQsIQn0KJqtoJceACnM
+Dyb6WSElwJ76IlNnbaumVgj9Nx1p/u9vKYkaZnvH/7E08da+IYHjvE8TAqdLnRMF
+RkCISqrb+2T4a/tdoLEMGrhMq29p/gtVTrM4H5ELcXceETlUmmJR6m2oXg1KkfvY
+vl2T6EPzShH7Mc8UGhyNMRuZMeArgQGRb9q6yx9RIVUpP0xx49ApQd6gANoH7V7J
+rzJhbVX49S1GAzQz+y4eqnz+0jBNQMztduz2ve01yNizRlaqLFOEVkWwo/Y1ZpPa
+jBc5wSl8mcULc8H5FtBX/FdZBq85n6lRNQvH
+-----END CERTIFICATE-----
diff --git a/tests/certificate-authority/newcerts/1008.pem b/tests/certificate-authority/newcerts/1008.pem
new file mode 100644
index 0000000000000..85687bdfd30ba
--- /dev/null
+++ b/tests/certificate-authority/newcerts/1008.pem
@@ -0,0 +1,110 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 4104 (0x1008)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: CN=foobar
+        Validity
+            Not Before: May 10 15:50:19 2023 GMT
+            Not After : Feb 22 15:50:19 2297 GMT
+        Subject: CN=proxy-localhost-SAN
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:cc:15:c9:85:06:43:47:bd:46:9f:4f:03:1a:e0:
+                    6e:94:13:4e:b0:30:ea:88:ca:3a:e4:39:92:12:c1:
+                    77:51:8c:0d:3c:b9:26:5c:2f:dc:fc:b1:5a:bf:0e:
+                    47:ff:09:60:30:79:8e:55:26:fe:d0:a1:ed:9f:6d:
+                    8a:6a:06:85:f0:d0:dc:94:a6:54:a1:a6:c9:3e:57:
+                    d5:69:7d:e9:25:c1:ef:6b:77:e1:62:76:d8:e4:54:
+                    91:40:bc:0b:11:74:b8:30:bb:d4:02:77:d6:bd:d2:
+                    d0:e7:ad:df:7d:98:96:74:42:ad:53:b3:88:c8:dc:
+                    1d:db:51:63:84:ee:7e:85:73:14:5e:d4:c8:f0:01:
+                    5f:67:52:ed:94:87:f7:d6:aa:28:8b:2c:84:98:8c:
+                    b9:91:b5:38:99:80:5d:b3:d4:db:95:96:09:ef:1d:
+                    a1:6f:86:c8:17:86:f7:0a:1e:72:3b:50:8c:53:e5:
+                    ce:d4:8c:cf:cc:81:3d:46:55:ff:65:25:0b:36:31:
+                    31:a6:22:27:47:96:59:38:c1:cd:66:a6:9a:83:98:
+                    dc:b8:2e:10:8d:ba:45:ae:aa:20:6e:e3:0b:bd:ec:
+                    e6:63:b5:40:55:d4:fe:97:b1:f1:8d:9a:c0:a2:46:
+                    8e:a3:ed:a0:1b:ed:40:b0:00:a5:28:f9:da:03:bd:
+                    c1:a9
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Cert Type: 
+                SSL Server
+            Netscape Comment: 
+                OpenSSL Generated Server Certificate
+            X509v3 Subject Key Identifier: 
+                C5:33:73:67:03:B7:51:08:F4:BD:D3:CD:4F:DC:CF:83:11:53:AD:39
+            X509v3 Subject Alternative Name: 
+                DNS:localhost, IP Address:127.0.0.1
+            X509v3 Authority Key Identifier: 
+                keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27
+                DirName:/CN=foobar
+                serial:D7:E2:87:4F:A0:79:E2:0C
+
+            X509v3 Key Usage: critical
+                Digital Signature, Key Encipherment
+            X509v3 Extended Key Usage: 
+                TLS Web Server Authentication
+    Signature Algorithm: sha256WithRSAEncryption
+         43:ef:67:29:9a:0c:53:97:7c:fc:72:73:6c:8d:48:78:4e:ec:
+         e3:14:9d:d9:1e:83:4c:d6:f0:56:e9:c4:d8:de:f5:54:fb:a5:
+         3b:ff:59:23:75:26:74:f0:86:90:d0:4d:41:25:03:87:e0:60:
+         a4:9b:33:3d:bd:1c:79:b8:db:86:1c:38:09:26:0d:80:3e:f9:
+         1e:28:11:0d:3d:6b:1e:1a:7a:9a:fa:fc:18:22:7f:fd:46:55:
+         c2:2f:56:5c:5c:8a:45:f2:74:7a:e4:6c:d0:e0:ea:ec:74:b7:
+         0d:a8:f3:ca:18:cf:a4:be:a0:e0:4a:32:ca:15:7e:5d:06:56:
+         b7:71:7c:e0:dc:19:fa:be:3e:94:84:20:be:96:34:61:0b:f0:
+         d1:d6:31:49:0b:b0:20:b8:f9:5c:49:08:13:9b:45:c0:6f:58:
+         16:81:0b:0c:f8:66:38:58:83:d4:b0:bc:14:35:8d:e2:1d:d5:
+         2d:ea:02:ae:42:e1:88:22:5a:b0:cf:e5:31:b1:cb:d3:e9:d2:
+         5e:88:55:bd:62:ac:85:aa:4e:fc:18:6b:65:f9:9e:fc:93:27:
+         0c:c6:29:aa:f0:64:6e:72:dc:d9:95:ae:38:ae:64:9e:c6:44:
+         8a:0b:0f:0e:d4:69:7e:79:e0:46:d0:75:96:2a:1a:60:af:30:
+         23:dc:d2:67:0d:08:2a:9d:58:29:09:1e:c8:08:d5:3a:88:2d:
+         1a:dc:47:dc:5d:bd:0d:5c:54:f1:5d:5a:6d:0d:de:bc:18:67:
+         2d:dd:1b:fe:8b:0e:03:19:b0:0f:f2:59:69:d0:7a:4f:a1:33:
+         74:f7:22:ef:ff:90:e1:4b:8e:ac:13:00:6f:00:9b:55:83:d2:
+         96:db:a8:81:c9:a9:8d:c6:a6:21:3d:14:d3:43:71:28:c6:ea:
+         6d:2d:91:b9:58:bf:ec:18:75:c4:8c:10:43:88:60:08:c0:bb:
+         9d:fb:90:80:1e:d5:a3:ea:e7:8a:16:f7:f4:d7:cb:35:93:03:
+         55:e4:cc:58:31:1e:df:6e:e4:1b:6e:ad:3a:76:56:e5:8b:4e:
+         d9:71:af:11:92:a7:7a:e2:66:cc:d2:73:f3:ec:e8:3b:67:f0:
+         6a:31:10:82:e8:c4:1e:ae:c3:54:a7:e2:42:86:fe:43:75:ad:
+         ef:83:d7:1c:2f:91:94:1c:57:9d:1c:43:94:b1:47:b2:6c:96:
+         fd:83:69:0f:6c:e2:18:9b:65:8e:71:08:01:b3:73:46:aa:3c:
+         2e:07:14:cd:03:ae:dc:5a:51:da:c5:41:53:cc:f5:fc:c8:db:
+         4e:76:27:99:9a:ec:40:68:07:d6:10:e1:f9:68:6b:5d:52:95:
+         3d:01:f4:a7:40:11:61:0a
+-----BEGIN CERTIFICATE-----
+MIIEpzCCAo+gAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
+YmFyMCAXDTIzMDUxMDE1NTAxOVoYDzIyOTcwMjIyMTU1MDE5WjAeMRwwGgYDVQQD
+DBNwcm94eS1sb2NhbGhvc3QtU0FOMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAzBXJhQZDR71Gn08DGuBulBNOsDDqiMo65DmSEsF3UYwNPLkmXC/c/LFa
+vw5H/wlgMHmOVSb+0KHtn22KagaF8NDclKZUoabJPlfVaX3pJcHva3fhYnbY5FSR
+QLwLEXS4MLvUAnfWvdLQ563ffZiWdEKtU7OIyNwd21FjhO5+hXMUXtTI8AFfZ1Lt
+lIf31qooiyyEmIy5kbU4mYBds9TblZYJ7x2hb4bIF4b3Ch5yO1CMU+XO1IzPzIE9
+RlX/ZSULNjExpiInR5ZZOMHNZqaag5jcuC4QjbpFrqogbuMLvezmY7VAVdT+l7Hx
+jZrAokaOo+2gG+1AsAClKPnaA73BqQIDAQABo4H5MIH2MAkGA1UdEwQCMAAwEQYJ
+YIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRl
+ZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFMUzc2cDt1EI9L3TzU/cz4MR
+U605MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATBBBgNVHSMEOjA4gBRXC+nL
+I+i/Rz5Qej9FfqEYQ50VJ6EVpBMwETEPMA0GA1UEAwwGZm9vYmFyggkA1+KHT6B5
+4gwwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3
+DQEBCwUAA4ICAQBD72cpmgxTl3z8cnNsjUh4TuzjFJ3ZHoNM1vBW6cTY3vVU+6U7
+/1kjdSZ08IaQ0E1BJQOH4GCkmzM9vRx5uNuGHDgJJg2APvkeKBENPWseGnqa+vwY
+In/9RlXCL1ZcXIpF8nR65GzQ4OrsdLcNqPPKGM+kvqDgSjLKFX5dBla3cXzg3Bn6
+vj6UhCC+ljRhC/DR1jFJC7AguPlcSQgTm0XAb1gWgQsM+GY4WIPUsLwUNY3iHdUt
+6gKuQuGIIlqwz+UxscvT6dJeiFW9YqyFqk78GGtl+Z78kycMximq8GRuctzZla44
+rmSexkSKCw8O1Gl+eeBG0HWWKhpgrzAj3NJnDQgqnVgpCR7ICNU6iC0a3EfcXb0N
+XFTxXVptDd68GGct3Rv+iw4DGbAP8llp0HpPoTN09yLv/5DhS46sEwBvAJtVg9KW
+26iByamNxqYhPRTTQ3EoxuptLZG5WL/sGHXEjBBDiGAIwLud+5CAHtWj6ueKFvf0
+18s1kwNV5MxYMR7fbuQbbq06dlbli07Zca8Rkqd64mbM0nPz7Og7Z/BqMRCC6MQe
+rsNUp+JChv5Dda3vg9ccL5GUHFedHEOUsUeybJb9g2kPbOIYm2WOcQgBs3NGqjwu
+BxTNA67cWlHaxUFTzPX8yNtOdieZmuxAaAfWEOH5aGtdUpU9AfSnQBFhCg==
+-----END CERTIFICATE-----
diff --git a/tests/certificate-authority/openssl.cnf b/tests/certificate-authority/openssl.cnf
index 9c8585edc9a1f..f7a23b3b33f6d 100644
--- a/tests/certificate-authority/openssl.cnf
+++ b/tests/certificate-authority/openssl.cnf
@@ -27,7 +27,7 @@ default_ca = CA_default
 
 [ CA_default ]
 # Directory and file locations.
-dir               = $ENV::CA_HOME
+dir               = .
 certs             = $dir/certs
 crl_dir           = $dir/crl
 new_certs_dir     = $dir/newcerts
@@ -92,12 +92,25 @@ authorityKeyIdentifier = keyid,issuer
 keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
 extendedKeyUsage = clientAuth, emailProtection
 
-[ server_cert ]
+[ broker_cert ]
 # Extensions for server certificates (`man x509v3_config`).
 basicConstraints = CA:FALSE
 nsCertType = server
 nsComment = "OpenSSL Generated Server Certificate"
 subjectKeyIdentifier = hash
+# The unresolvable address is used for SNI testing
+subjectAltName = DNS:localhost, DNS:unresolvable-broker-address, IP:127.0.0.1
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+
+[ proxy_cert ]
+# Extensions for server certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = server
+nsComment = "OpenSSL Generated Server Certificate"
+subjectKeyIdentifier = hash
+subjectAltName = DNS:localhost, IP:127.0.0.1
 authorityKeyIdentifier = keyid,issuer:always
 keyUsage = critical, digitalSignature, keyEncipherment
 extendedKeyUsage = serverAuth
diff --git a/tests/certificate-authority/serial b/tests/certificate-authority/serial
index fb35a14c02716..6cb3869343bf8 100644
--- a/tests/certificate-authority/serial
+++ b/tests/certificate-authority/serial
@@ -1 +1 @@
-1007
+1009
diff --git a/tests/certificate-authority/server-keys/broker.cert.pem b/tests/certificate-authority/server-keys/broker.cert.pem
index b5c7a5dc709a1..4237719f20ebd 100644
--- a/tests/certificate-authority/server-keys/broker.cert.pem
+++ b/tests/certificate-authority/server-keys/broker.cert.pem
@@ -1,27 +1,111 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 4103 (0x1007)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: CN=foobar
+        Validity
+            Not Before: May 10 15:50:18 2023 GMT
+            Not After : Feb 22 15:50:18 2297 GMT
+        Subject: CN=broker-localhost-SAN
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:de:d1:da:bb:91:b3:16:c4:b2:e8:89:30:9e:c1:
+                    5e:0b:cf:db:c4:c3:d9:b1:af:40:a5:0b:38:36:1b:
+                    14:fe:0f:22:9c:e6:59:6a:15:5b:db:f6:f7:f3:a5:
+                    02:29:94:7a:d2:0c:67:ad:aa:63:62:7e:fc:58:11:
+                    29:48:b8:3c:91:b2:73:7e:12:6b:f2:ea:36:77:0f:
+                    15:9b:46:95:ce:73:15:8d:c8:d9:97:57:03:90:33:
+                    2d:7d:f3:ee:e5:01:6d:d8:c6:da:ab:07:b9:dd:1c:
+                    e0:4b:ce:6a:de:a8:d2:e3:c1:52:6d:83:3a:0a:f0:
+                    ed:cf:f7:56:6a:87:0e:73:e3:12:82:2b:65:ab:d8:
+                    a9:44:5b:4a:2f:a5:92:94:32:f1:a1:e4:af:18:0f:
+                    0f:18:60:cd:f7:d0:9d:03:9f:d7:e9:a8:60:54:bb:
+                    3b:9a:05:db:fd:38:04:3c:b4:23:41:16:6c:7c:3b:
+                    d9:b6:e0:2f:bd:cb:62:55:1b:e8:d0:8f:43:76:ef:
+                    55:86:cf:25:c3:bc:ae:e3:46:50:89:f7:71:ad:06:
+                    5e:28:e6:f6:f0:76:27:ea:7e:1b:67:53:39:26:20:
+                    19:18:82:b1:11:5f:ea:91:c2:e3:d3:f6:5a:c7:fd:
+                    61:a2:92:de:7d:7c:da:6d:e8:bf:39:52:10:31:60:
+                    4b:e1
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Cert Type: 
+                SSL Server
+            Netscape Comment: 
+                OpenSSL Generated Server Certificate
+            X509v3 Subject Key Identifier: 
+                17:07:3B:AA:85:83:B5:04:83:EC:B2:6C:1E:3A:F0:F5:59:AA:61:28
+            X509v3 Subject Alternative Name: 
+                DNS:localhost, DNS:unresolvable-broker-address, IP Address:127.0.0.1
+            X509v3 Authority Key Identifier: 
+                keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27
+                DirName:/CN=foobar
+                serial:D7:E2:87:4F:A0:79:E2:0C
+
+            X509v3 Key Usage: critical
+                Digital Signature, Key Encipherment
+            X509v3 Extended Key Usage: 
+                TLS Web Server Authentication
+    Signature Algorithm: sha256WithRSAEncryption
+         e4:27:61:e2:0f:b6:a0:ca:9f:ce:e3:53:0b:44:ab:86:a1:e2:
+         4d:88:e1:7d:2e:b0:aa:32:96:2b:3d:da:60:70:6a:c3:62:c5:
+         76:f2:8f:0d:16:31:f2:ad:e5:2f:43:f3:cb:e4:fa:95:6c:20:
+         81:33:1a:c7:5a:55:57:c9:ab:ca:66:45:30:58:00:db:e8:51:
+         c9:2c:a9:72:c1:18:f5:01:87:9f:73:20:85:6c:e5:6c:3f:c9:
+         67:b4:f0:20:e5:ed:e2:4a:08:0b:af:68:43:e5:a9:c7:e1:39:
+         e8:b5:49:cb:47:4a:6d:e5:16:ae:88:92:13:85:8e:42:1e:0a:
+         eb:59:ed:a7:c1:9b:bc:4b:7b:99:f8:1d:f0:d7:1d:90:c9:cf:
+         86:6a:d3:10:d0:36:e4:f5:b9:33:79:c7:a2:68:31:f7:bb:8d:
+         1e:d6:33:79:bd:e7:0e:4f:4d:e9:2e:15:04:4f:6b:4b:2e:93:
+         28:72:d1:0e:aa:ee:e6:ef:68:be:58:2b:cc:56:01:27:16:f9:
+         34:8e:66:86:27:0a:b0:fb:32:56:a9:8a:d9:6f:b1:86:bd:ba:
+         fd:50:6c:d5:b2:54:e7:4e:c6:2d:19:88:a9:89:2c:ef:be:08:
+         0d:2b:49:91:0b:09:42:64:06:a3:9d:d7:94:ed:e8:74:74:48:
+         43:57:41:6f:e5:06:98:46:1d:c5:60:9c:69:f8:fb:fe:a6:01:
+         4a:35:be:21:36:c2:a3:44:c8:c4:2c:21:09:f4:28:9a:ad:a0:
+         97:1e:00:29:cc:0f:26:fa:59:21:25:c0:9e:fa:22:53:67:6d:
+         ab:a6:56:08:fd:37:1d:69:fe:ef:6f:29:89:1a:66:7b:c7:ff:
+         b1:34:f1:d6:be:21:81:e3:bc:4f:13:02:a7:4b:9d:13:05:46:
+         40:88:4a:aa:db:fb:64:f8:6b:fb:5d:a0:b1:0c:1a:b8:4c:ab:
+         6f:69:fe:0b:55:4e:b3:38:1f:91:0b:71:77:1e:11:39:54:9a:
+         62:51:ea:6d:a8:5e:0d:4a:91:fb:d8:be:5d:93:e8:43:f3:4a:
+         11:fb:31:cf:14:1a:1c:8d:31:1b:99:31:e0:2b:81:01:91:6f:
+         da:ba:cb:1f:51:21:55:29:3f:4c:71:e3:d0:29:41:de:a0:00:
+         da:07:ed:5e:c9:af:32:61:6d:55:f8:f5:2d:46:03:34:33:fb:
+         2e:1e:aa:7c:fe:d2:30:4d:40:cc:ed:76:ec:f6:bd:ed:35:c8:
+         d8:b3:46:56:aa:2c:53:84:56:45:b0:a3:f6:35:66:93:da:8c:
+         17:39:c1:29:7c:99:c5:0b:73:c1:f9:16:d0:57:fc:57:59:06:
+         af:39:9f:a9:51:35:0b:c7
 -----BEGIN CERTIFICATE-----
-MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD
-DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
-DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63
-EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O
-OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01
-+ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X
-aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s
-bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw
-ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
-ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA
-DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx
-DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV
-HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ
-ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y
-ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN
-zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9
-spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur
-x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0
-fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d
-mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS
-efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp
-0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C
-cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK
-3N2cGJWRyvpvvmbP2hKCa4eLgEc=
+MIIExzCCAq+gAwIBAgICEAcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
+YmFyMCAXDTIzMDUxMDE1NTAxOFoYDzIyOTcwMjIyMTU1MDE4WjAfMR0wGwYDVQQD
+DBRicm9rZXItbG9jYWxob3N0LVNBTjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAN7R2ruRsxbEsuiJMJ7BXgvP28TD2bGvQKULODYbFP4PIpzmWWoVW9v2
+9/OlAimUetIMZ62qY2J+/FgRKUi4PJGyc34Sa/LqNncPFZtGlc5zFY3I2ZdXA5Az
+LX3z7uUBbdjG2qsHud0c4EvOat6o0uPBUm2DOgrw7c/3VmqHDnPjEoIrZavYqURb
+Si+lkpQy8aHkrxgPDxhgzffQnQOf1+moYFS7O5oF2/04BDy0I0EWbHw72bbgL73L
+YlUb6NCPQ3bvVYbPJcO8ruNGUIn3ca0GXijm9vB2J+p+G2dTOSYgGRiCsRFf6pHC
+49P2Wsf9YaKS3n182m3ovzlSEDFgS+ECAwEAAaOCARcwggETMAkGA1UdEwQCMAAw
+EQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVy
+YXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBcHO6qFg7UEg+yybB46
+8PVZqmEoMDcGA1UdEQQwMC6CCWxvY2FsaG9zdIIbdW5yZXNvbHZhYmxlLWJyb2tl
+ci1hZGRyZXNzhwR/AAABMEEGA1UdIwQ6MDiAFFcL6csj6L9HPlB6P0V+oRhDnRUn
+oRWkEzARMQ8wDQYDVQQDDAZmb29iYXKCCQDX4odPoHniDDAOBgNVHQ8BAf8EBAMC
+BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAOQnYeIP
+tqDKn87jUwtEq4ah4k2I4X0usKoylis92mBwasNixXbyjw0WMfKt5S9D88vk+pVs
+IIEzGsdaVVfJq8pmRTBYANvoUcksqXLBGPUBh59zIIVs5Ww/yWe08CDl7eJKCAuv
+aEPlqcfhOei1SctHSm3lFq6IkhOFjkIeCutZ7afBm7xLe5n4HfDXHZDJz4Zq0xDQ
+NuT1uTN5x6JoMfe7jR7WM3m95w5PTekuFQRPa0sukyhy0Q6q7ubvaL5YK8xWAScW
++TSOZoYnCrD7MlapitlvsYa9uv1QbNWyVOdOxi0ZiKmJLO++CA0rSZELCUJkBqOd
+15Tt6HR0SENXQW/lBphGHcVgnGn4+/6mAUo1viE2wqNEyMQsIQn0KJqtoJceACnM
+Dyb6WSElwJ76IlNnbaumVgj9Nx1p/u9vKYkaZnvH/7E08da+IYHjvE8TAqdLnRMF
+RkCISqrb+2T4a/tdoLEMGrhMq29p/gtVTrM4H5ELcXceETlUmmJR6m2oXg1KkfvY
+vl2T6EPzShH7Mc8UGhyNMRuZMeArgQGRb9q6yx9RIVUpP0xx49ApQd6gANoH7V7J
+rzJhbVX49S1GAzQz+y4eqnz+0jBNQMztduz2ve01yNizRlaqLFOEVkWwo/Y1ZpPa
+jBc5wSl8mcULc8H5FtBX/FdZBq85n6lRNQvH
 -----END CERTIFICATE-----
diff --git a/tests/certificate-authority/server-keys/broker.csr.pem b/tests/certificate-authority/server-keys/broker.csr.pem
index d2342595eb254..9d28c52be7909 100644
--- a/tests/certificate-authority/server-keys/broker.csr.pem
+++ b/tests/certificate-authority/server-keys/broker.csr.pem
@@ -1,15 +1,15 @@
 -----BEGIN CERTIFICATE REQUEST-----
-MIICaDCCAVACAQAwIzEhMB8GA1UEAwwYYnJva2VyLnB1bHNhci5hcGFjaGUub3Jn
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0KLioWWoeITAqpoOGkuU
-YUBv2NYAPsj/QxhfZpcN+bX4X1rOtxLzYxCuC/2VkhFeOS/8HOmFdeEZjOmydFf9
-L16OMZiEm6qbTQ/9pESLKZC2UORdDjkF1YnKmSznOooGUIUYPADQ2HUyVExeuQ6m
-rl+Dibd2DIRdhUre40LZY9OiEAWtNfgETvTqLTCbQWPryFOW1eicnX4JIUB/ayAy
-OwDHmOY9NoBpCSa3pX5vlqDrFHs/l2qkzd/7RfPjAXBQdxtOLNanNgiTyTN2Bsb8
-RLR6kxiNeLfSfKeTu1/SR6XGPF9NLG8WuBRmyoHw6G25CfhoG5VgpAHCzfebCk6r
-RwIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAHVVGKnfqBDmu+e5MWK9i0ja/JFv
-dhST705gdKDOPc7MXDVr+zJZKgvnDtzDrWTe7Zk0p7xQf3kc773eYCdlznX+J1Fw
-EfIHXQTBZRZxmHnYqc012i5tshvEOS0o61ZEgxz8hxGLwGlRaIcy+qt927fscpQ5
-7VEnlxzD4YeHwryIXH5hOr/J1OmlL58Fxwh2NJfso7ErRuHW44XK4qdwWCQs/nVN
-EQyV6RCbaiRq9Ks4j3FwtqmfgzMB1+T3L+CiuhPol2/rZwD3o5j7SP8ZGxC15Tzi
-wHG71H0wp1CY+tkAcvm2zmoHR9z1SD84raZLYJVRgUio7myW/DVBqPxCSvU=
+MIICZDCCAUwCAQAwHzEdMBsGA1UEAwwUYnJva2VyLWxvY2FsaG9zdC1TQU4wggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDe0dq7kbMWxLLoiTCewV4Lz9vE
+w9mxr0ClCzg2GxT+DyKc5llqFVvb9vfzpQIplHrSDGetqmNifvxYESlIuDyRsnN+
+Emvy6jZ3DxWbRpXOcxWNyNmXVwOQMy198+7lAW3YxtqrB7ndHOBLzmreqNLjwVJt
+gzoK8O3P91Zqhw5z4xKCK2Wr2KlEW0ovpZKUMvGh5K8YDw8YYM330J0Dn9fpqGBU
+uzuaBdv9OAQ8tCNBFmx8O9m24C+9y2JVG+jQj0N271WGzyXDvK7jRlCJ93GtBl4o
+5vbwdifqfhtnUzkmIBkYgrERX+qRwuPT9lrH/WGikt59fNpt6L85UhAxYEvhAgMB
+AAGgADANBgkqhkiG9w0BAQsFAAOCAQEAcLkzWe6zgkVpk4OUlCv1HUqmntiBHdmh
+24v3OKhQjyMs+m/srBe6r+lBdZLnbSH5fq7eToEwRMt/vNirsXCaxGGVOUnThStt
+Z/rR45bpJl1TomXwEG9xHq7yOaHVfxkNZgaHCu6BZ1ZjYgfqWffFf9hcqAJ3xZm0
+XMPfJs9i/TRHCsdUge1BHZrxD/fzriWM7k89XktY+0zSqGfdhOXE0FO30bnC4mJG
+vZXY5reyIuRlFBpnDUtuYBaSfbUYguaSUYIoHOUsQrmMSqPLJUyY1zNm1t5f1jIx
+ZaK8NEIq2AHHqEx7I/7+lVRRWa9IjdqWIT9KD5TraOML9oS74sto2w==
 -----END CERTIFICATE REQUEST-----
diff --git a/tests/certificate-authority/server-keys/broker.key-pk8.pem b/tests/certificate-authority/server-keys/broker.key-pk8.pem
index 2b51d015b8ace..dd9fa523e8ede 100644
--- a/tests/certificate-authority/server-keys/broker.key-pk8.pem
+++ b/tests/certificate-authority/server-keys/broker.key-pk8.pem
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq
-mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM
-6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU
-TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh
-QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ
-M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN
-95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j
-WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN
-wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y
-3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK
-r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv
-yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP
-eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj
-bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF
-M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP
-PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9
-T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2
-yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP
-UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat
-V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt
-6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm
-3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ
-2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA
-G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+
-I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI
-5qJ7jxgksHWji4lYfL4r4J6Zaw==
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe0dq7kbMWxLLo
+iTCewV4Lz9vEw9mxr0ClCzg2GxT+DyKc5llqFVvb9vfzpQIplHrSDGetqmNifvxY
+ESlIuDyRsnN+Emvy6jZ3DxWbRpXOcxWNyNmXVwOQMy198+7lAW3YxtqrB7ndHOBL
+zmreqNLjwVJtgzoK8O3P91Zqhw5z4xKCK2Wr2KlEW0ovpZKUMvGh5K8YDw8YYM33
+0J0Dn9fpqGBUuzuaBdv9OAQ8tCNBFmx8O9m24C+9y2JVG+jQj0N271WGzyXDvK7j
+RlCJ93GtBl4o5vbwdifqfhtnUzkmIBkYgrERX+qRwuPT9lrH/WGikt59fNpt6L85
+UhAxYEvhAgMBAAECggEAbQ4xDFTHXoFvPzjGPy1NJmLZoXhp9/lanmzbWj/vClnG
+Cx0C7lT93K8HtIwyfr9ZTa0coXcfpXmZcFEV762cl4LL3AyQIRhZB/SuEo19jMnu
+5rJDLTs9Vzp1LYxShGsqpErPg54IbhxP+0pQLCJc9XQNL+RmaCx7eKoJ9aGchULY
+BdbCRctYhHaq/AC2qNYZF41Ys9zdNN0/2NnRqfgaaAj9hUzu3LlaJX1TWHbgikLo
+QdmRNmTMMxfLl2kyoweDEC6MdSKCbcdDyM46Va3yY3KTuAdjP6x1ALzAarBDj6zb
+a0Vp7g80OcKjmLYt8rFImjwb7D9EanOy2GkK2CwhAQKBgQD8v4gOTeNYC4Xo6Rti
+psv+QCuH8hiLdee4KFdzqthlELDfhncDKXfwcZI3PME/aBWvvAn+rokl/UfCm5nQ
+fwXW3MYyNpmk0HJjWAQcexsdHCM9I0CgEp8uInn+8EFqlYVh2ltoQLhGuIwOqqPk
+3cQV8ImW+CmqmWYzbSZw97OqqQKBgQDhr7+oOf+sly0MVf5WixHTURJAU6p2VyTt
+aNsNiuLN/W3Vax9Ql9HEm3RPx2SxFEIxllPcwb1Vms/ONmvT1t3xWGp7TIOX/0/m
+uhNIG2/Bcr47NgWjhiV4zE/TfawkcP7/MujUxl2/zm7RHLrU2bmi8rV+PGqlVE0w
+v7iKj4bSeQKBgHkI34a6Fdzb58yZlNuxNI8U+8OmU8q1M7ok13w0nFwJminwop2J
+Bj7GpFZ/aauLlJcLXV3xBwyCNhMjoI0PxyQVpXP2Ya1jhOO+Cnn5GgrepqFoeFIv
+mLrnF7TWKP15jN5HSu6pz5VOWwPLA6Fd8cDv53O8c3eW7jJCWt5OQGPBAoGALwbI
+EO3E8NmvcVqZ3L6twDKscur8IhyWfUHUI0ZFbFbahBYGOGzqMOWTnuwVdzCZemuw
+nddg9G2Fz5pXbZTgOmIKDhcrdIimxZUQX34YE18tdHkVQ7W4KSuplpAhRpalC9g3
+295ZupXxUXGDHMchf2rDlsJQFpMyYm4Qrg6qMUECgYBhBG/iLvE6/Z1oh6eky6WU
+Dx7nL+FDDu3JbNAWs6RMDs9fE+7iDEj8MxZL0PUnVMsQvf43Ew1boyIGwlNtdRZ5
+dLaKFAgJ8YbqIptnFfGQ/8vKhK+x4FWzEd3kuRzQZPVRVV5Op6bRp1Kb8NMPCQXb
+72qifYUaI7uirULNSit9wg==
 -----END PRIVATE KEY-----
diff --git a/tests/certificate-authority/server-keys/broker.key.pem b/tests/certificate-authority/server-keys/broker.key.pem
index dc22667ab47a5..5c20238c7b9c9 100644
--- a/tests/certificate-authority/server-keys/broker.key.pem
+++ b/tests/certificate-authority/server-keys/broker.key.pem
@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEA0KLioWWoeITAqpoOGkuUYUBv2NYAPsj/QxhfZpcN+bX4X1rO
-txLzYxCuC/2VkhFeOS/8HOmFdeEZjOmydFf9L16OMZiEm6qbTQ/9pESLKZC2UORd
-DjkF1YnKmSznOooGUIUYPADQ2HUyVExeuQ6mrl+Dibd2DIRdhUre40LZY9OiEAWt
-NfgETvTqLTCbQWPryFOW1eicnX4JIUB/ayAyOwDHmOY9NoBpCSa3pX5vlqDrFHs/
-l2qkzd/7RfPjAXBQdxtOLNanNgiTyTN2Bsb8RLR6kxiNeLfSfKeTu1/SR6XGPF9N
-LG8WuBRmyoHw6G25CfhoG5VgpAHCzfebCk6rRwIDAQABAoIBAQCxNXjLX55N52wC
-O+G4ofOA+wtFINeKfIyb/6qgZ7MeI1q4G1gzFayWu0cpthUBwTDH4CMCp7KNIkni
-b2j8+vN49ZZPN92CntFUDj4A9ZtCDcFdSepkNzjmXCgyHHq2NSAR6kpgRM0e3cSG
-52sZQgF/YG9IXQSuswsAOUzBuTkOWN3BjpnryuYtj4no4aa7gPIexOPD9QE2hh1G
-Yn/WA+9uKIugHIpUPB/w5To2U75Ziq+Gid9H+9jcBjPBHHreQhNsMn5QBrWNJPpw
-oWJD0X8ueLuuGf0/QedwL53hWo/I78hXqWYEU7NnFoRJdCkBrgR0OPiHjnzLdi3m
-4b41lvGJAoGBAOwV1bJ6jxykQ3fMD3jZKJWOBY3auMLvWnfg2hYmXCbr+7oK/iqZ
-ZbaunuGlN+sxHQWHT6f+ijPxxhP1426NAhCzVCrPzcvR/1CXPAw21RJqBd1xoiGy
-iqtU/DU10xExZ8KxpARQ/J5e2E7DhTOBWZkqJNDIYZKRFCwKsQUumKULAoGBAOI8
-TNXoE1Ulo063oNlHWKNudO4ymu5mjz0baD7DSc1jWzqX49upd//KXeqGClnJOrHu
-kcSFMVK5N8sITSSCDr1hAu02jh9XPU+DUDXMHqgVvfgNceYMs09eIA2gX5eOYLPN
-ryPpHKVb/8Njdx2TkGV98gFKlqX8NssNbkA17YA1AoGBAImD1mwV05KdY+c9560+
-4NLsXPPZ+VEi1pGJWb8ZLQhG4OPnT1FE9OhKIqXQ9Rd8DimQPaffLmU/aV9rFMTq
-Qc+0hUxCqt8UPSGWVhXPQqFtPGlWrVfvhbJy2E2uQnl7S7Mughk5i0w8dsa6QLZV
-m/kJgVRTw/H5q8RTnfUkxSktAoGALes7VTdzpRasnO+ckD1w+ksTJL3UwIrEjAyi
-DgspUazW+X+W2iD9oEd85HssYqj7Zt4bbB30suuxLV6T+dMEFuA37dni4u2cHiVz
-BFhX6JnxIwXnlmABjRx3uQxSGrrlWdlnsQxBqjaQJSJtLO2ko1T659Qx26LjpDcn
-p5TKjfkCgYEA1WFYZPcInUoUpwplABt9JqFw4I93kagn+U+zRK/Rlw9UAQ/60Ej4
-eWoJw1eyGC6Ve/8JGCyO5fce5g8LviPZkV5MgJk1RHLS03V0B7dI3SWuUn1GlRA1
-M0G69H1I5JsrYU76mfNPfndxCz3iyOaie48YJLB1o4uJWHy+K+CemWs=
+MIIEogIBAAKCAQEA3tHau5GzFsSy6IkwnsFeC8/bxMPZsa9ApQs4NhsU/g8inOZZ
+ahVb2/b386UCKZR60gxnrapjYn78WBEpSLg8kbJzfhJr8uo2dw8Vm0aVznMVjcjZ
+l1cDkDMtffPu5QFt2Mbaqwe53RzgS85q3qjS48FSbYM6CvDtz/dWaocOc+MSgitl
+q9ipRFtKL6WSlDLxoeSvGA8PGGDN99CdA5/X6ahgVLs7mgXb/TgEPLQjQRZsfDvZ
+tuAvvctiVRvo0I9Ddu9Vhs8lw7yu40ZQifdxrQZeKOb28HYn6n4bZ1M5JiAZGIKx
+EV/qkcLj0/Zax/1hopLefXzabei/OVIQMWBL4QIDAQABAoIBAG0OMQxUx16Bbz84
+xj8tTSZi2aF4aff5Wp5s21o/7wpZxgsdAu5U/dyvB7SMMn6/WU2tHKF3H6V5mXBR
+Fe+tnJeCy9wMkCEYWQf0rhKNfYzJ7uayQy07PVc6dS2MUoRrKqRKz4OeCG4cT/tK
+UCwiXPV0DS/kZmgse3iqCfWhnIVC2AXWwkXLWIR2qvwAtqjWGReNWLPc3TTdP9jZ
+0an4GmgI/YVM7ty5WiV9U1h24IpC6EHZkTZkzDMXy5dpMqMHgxAujHUigm3HQ8jO
+OlWt8mNyk7gHYz+sdQC8wGqwQ4+s22tFae4PNDnCo5i2LfKxSJo8G+w/RGpzsthp
+CtgsIQECgYEA/L+IDk3jWAuF6OkbYqbL/kArh/IYi3XnuChXc6rYZRCw34Z3Ayl3
+8HGSNzzBP2gVr7wJ/q6JJf1HwpuZ0H8F1tzGMjaZpNByY1gEHHsbHRwjPSNAoBKf
+LiJ5/vBBapWFYdpbaEC4RriMDqqj5N3EFfCJlvgpqplmM20mcPezqqkCgYEA4a+/
+qDn/rJctDFX+VosR01ESQFOqdlck7WjbDYrizf1t1WsfUJfRxJt0T8dksRRCMZZT
+3MG9VZrPzjZr09bd8Vhqe0yDl/9P5roTSBtvwXK+OzYFo4YleMxP032sJHD+/zLo
+1MZdv85u0Ry61Nm5ovK1fjxqpVRNML+4io+G0nkCgYB5CN+GuhXc2+fMmZTbsTSP
+FPvDplPKtTO6JNd8NJxcCZop8KKdiQY+xqRWf2mri5SXC11d8QcMgjYTI6CND8ck
+FaVz9mGtY4Tjvgp5+RoK3qahaHhSL5i65xe01ij9eYzeR0ruqc+VTlsDywOhXfHA
+7+dzvHN3lu4yQlreTkBjwQKBgC8GyBDtxPDZr3Famdy+rcAyrHLq/CIcln1B1CNG
+RWxW2oQWBjhs6jDlk57sFXcwmXprsJ3XYPRthc+aV22U4DpiCg4XK3SIpsWVEF9+
+GBNfLXR5FUO1uCkrqZaQIUaWpQvYN9veWbqV8VFxgxzHIX9qw5bCUBaTMmJuEK4O
+qjFBAoGAYQRv4i7xOv2daIenpMullA8e5y/hQw7tyWzQFrOkTA7PXxPu4gxI/DMW
+S9D1J1TLEL3+NxMNW6MiBsJTbXUWeXS2ihQICfGG6iKbZxXxkP/LyoSvseBVsxHd
+5Lkc0GT1UVVeTqem0adSm/DTDwkF2+9qon2FGiO7oq1CzUorfcI=
 -----END RSA PRIVATE KEY-----
diff --git a/tests/certificate-authority/server-keys/proxy.cert.pem b/tests/certificate-authority/server-keys/proxy.cert.pem
index 02caee5826346..85687bdfd30ba 100644
--- a/tests/certificate-authority/server-keys/proxy.cert.pem
+++ b/tests/certificate-authority/server-keys/proxy.cert.pem
@@ -1,27 +1,110 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 4104 (0x1008)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: CN=foobar
+        Validity
+            Not Before: May 10 15:50:19 2023 GMT
+            Not After : Feb 22 15:50:19 2297 GMT
+        Subject: CN=proxy-localhost-SAN
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:cc:15:c9:85:06:43:47:bd:46:9f:4f:03:1a:e0:
+                    6e:94:13:4e:b0:30:ea:88:ca:3a:e4:39:92:12:c1:
+                    77:51:8c:0d:3c:b9:26:5c:2f:dc:fc:b1:5a:bf:0e:
+                    47:ff:09:60:30:79:8e:55:26:fe:d0:a1:ed:9f:6d:
+                    8a:6a:06:85:f0:d0:dc:94:a6:54:a1:a6:c9:3e:57:
+                    d5:69:7d:e9:25:c1:ef:6b:77:e1:62:76:d8:e4:54:
+                    91:40:bc:0b:11:74:b8:30:bb:d4:02:77:d6:bd:d2:
+                    d0:e7:ad:df:7d:98:96:74:42:ad:53:b3:88:c8:dc:
+                    1d:db:51:63:84:ee:7e:85:73:14:5e:d4:c8:f0:01:
+                    5f:67:52:ed:94:87:f7:d6:aa:28:8b:2c:84:98:8c:
+                    b9:91:b5:38:99:80:5d:b3:d4:db:95:96:09:ef:1d:
+                    a1:6f:86:c8:17:86:f7:0a:1e:72:3b:50:8c:53:e5:
+                    ce:d4:8c:cf:cc:81:3d:46:55:ff:65:25:0b:36:31:
+                    31:a6:22:27:47:96:59:38:c1:cd:66:a6:9a:83:98:
+                    dc:b8:2e:10:8d:ba:45:ae:aa:20:6e:e3:0b:bd:ec:
+                    e6:63:b5:40:55:d4:fe:97:b1:f1:8d:9a:c0:a2:46:
+                    8e:a3:ed:a0:1b:ed:40:b0:00:a5:28:f9:da:03:bd:
+                    c1:a9
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Cert Type: 
+                SSL Server
+            Netscape Comment: 
+                OpenSSL Generated Server Certificate
+            X509v3 Subject Key Identifier: 
+                C5:33:73:67:03:B7:51:08:F4:BD:D3:CD:4F:DC:CF:83:11:53:AD:39
+            X509v3 Subject Alternative Name: 
+                DNS:localhost, IP Address:127.0.0.1
+            X509v3 Authority Key Identifier: 
+                keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27
+                DirName:/CN=foobar
+                serial:D7:E2:87:4F:A0:79:E2:0C
+
+            X509v3 Key Usage: critical
+                Digital Signature, Key Encipherment
+            X509v3 Extended Key Usage: 
+                TLS Web Server Authentication
+    Signature Algorithm: sha256WithRSAEncryption
+         43:ef:67:29:9a:0c:53:97:7c:fc:72:73:6c:8d:48:78:4e:ec:
+         e3:14:9d:d9:1e:83:4c:d6:f0:56:e9:c4:d8:de:f5:54:fb:a5:
+         3b:ff:59:23:75:26:74:f0:86:90:d0:4d:41:25:03:87:e0:60:
+         a4:9b:33:3d:bd:1c:79:b8:db:86:1c:38:09:26:0d:80:3e:f9:
+         1e:28:11:0d:3d:6b:1e:1a:7a:9a:fa:fc:18:22:7f:fd:46:55:
+         c2:2f:56:5c:5c:8a:45:f2:74:7a:e4:6c:d0:e0:ea:ec:74:b7:
+         0d:a8:f3:ca:18:cf:a4:be:a0:e0:4a:32:ca:15:7e:5d:06:56:
+         b7:71:7c:e0:dc:19:fa:be:3e:94:84:20:be:96:34:61:0b:f0:
+         d1:d6:31:49:0b:b0:20:b8:f9:5c:49:08:13:9b:45:c0:6f:58:
+         16:81:0b:0c:f8:66:38:58:83:d4:b0:bc:14:35:8d:e2:1d:d5:
+         2d:ea:02:ae:42:e1:88:22:5a:b0:cf:e5:31:b1:cb:d3:e9:d2:
+         5e:88:55:bd:62:ac:85:aa:4e:fc:18:6b:65:f9:9e:fc:93:27:
+         0c:c6:29:aa:f0:64:6e:72:dc:d9:95:ae:38:ae:64:9e:c6:44:
+         8a:0b:0f:0e:d4:69:7e:79:e0:46:d0:75:96:2a:1a:60:af:30:
+         23:dc:d2:67:0d:08:2a:9d:58:29:09:1e:c8:08:d5:3a:88:2d:
+         1a:dc:47:dc:5d:bd:0d:5c:54:f1:5d:5a:6d:0d:de:bc:18:67:
+         2d:dd:1b:fe:8b:0e:03:19:b0:0f:f2:59:69:d0:7a:4f:a1:33:
+         74:f7:22:ef:ff:90:e1:4b:8e:ac:13:00:6f:00:9b:55:83:d2:
+         96:db:a8:81:c9:a9:8d:c6:a6:21:3d:14:d3:43:71:28:c6:ea:
+         6d:2d:91:b9:58:bf:ec:18:75:c4:8c:10:43:88:60:08:c0:bb:
+         9d:fb:90:80:1e:d5:a3:ea:e7:8a:16:f7:f4:d7:cb:35:93:03:
+         55:e4:cc:58:31:1e:df:6e:e4:1b:6e:ad:3a:76:56:e5:8b:4e:
+         d9:71:af:11:92:a7:7a:e2:66:cc:d2:73:f3:ec:e8:3b:67:f0:
+         6a:31:10:82:e8:c4:1e:ae:c3:54:a7:e2:42:86:fe:43:75:ad:
+         ef:83:d7:1c:2f:91:94:1c:57:9d:1c:43:94:b1:47:b2:6c:96:
+         fd:83:69:0f:6c:e2:18:9b:65:8e:71:08:01:b3:73:46:aa:3c:
+         2e:07:14:cd:03:ae:dc:5a:51:da:c5:41:53:cc:f5:fc:c8:db:
+         4e:76:27:99:9a:ec:40:68:07:d6:10:e1:f9:68:6b:5d:52:95:
+         3d:01:f4:a7:40:11:61:0a
 -----BEGIN CERTIFICATE-----
-MIIEjzCCAnegAwIBAgICEAYwDQYJKoZIhvcNAQENBQAwETEPMA0GA1UEAwwGZm9v
-YmFyMCAXDTIyMTAxODEwMTQwMVoYDzIyOTYwODAyMTAxNDAxWjAiMSAwHgYDVQQD
-DBdwcm94eS5wdWxzYXIuYXBhY2hlLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEP
-ADCCAQoCggEBAPPnBnkHqKvXuv7BKOoQ8nAa7gEVAjzRANhOx2Yk3/JpN1/Ash48
-UltPjHtop1kXLrnjM3DahQuolz1A/N5sN2RGoe+/Y/aI/FRDF25yGzEoM/kwZDjm
-ejQj2Hb6YsupI+YYtPr5ZDSeIBvvlVurXfXJkZf5CXYeEjqr1pEpLpNCZoWoOiiC
-73/0KBoOToR5+akw+Db2Qr5FSz7AuTQ9KUZ1HZNl4xZBuEha6avESdRykH2XQzDs
-qMBVruByHbzO1pg/op4iOhqQ6DFu67veKjWzMLxKR7x/A8UOd9f9D3+pabBoU72b
-NqgwbKCnERoo3Y0ge1B1x7GORR7GHrWSKlUCAwEAAaOB3TCB2jAJBgNVHRMEAjAA
-MBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5l
-cmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1UdDgQWBBQqVR7lwaEgKHsI8+D8
-nNxPmgWZ7TBBBgNVHSMEOjA4gBRXC+nLI+i/Rz5Qej9FfqEYQ50VJ6EVpBMwETEP
-MA0GA1UEAwwGZm9vYmFyggkA1+KHT6B54gwwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud
-JQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBDQUAA4ICAQBtoQTZ5u6NpDIKHo6V
-yZqkRrMcg9J61zRm0tbf4D/iIsfWNiJrAWSudK4OgkUrXj4LFWKvzzcZtPltuUr5
-yODXZgz8lnyLbw6GyrKFU4Gpbr8Be30Y1yF7dfTV0yp5ZoIXNILfKhU3not1yL41
-0owaO7N0PyDAzQ7erPbbB9UG7xhYM5qFfAnevwX1rde12JHJULfeE9Ushuv+DcK5
-JmNvkRE+nB/dljsST9pW+zjBDuhwTiDZMPtUPyM0tPn6+x5zwF0pWFKhCkO8lVhr
-TxCG/bMF3j/0MxjQvDvcijJFHaZqLHsw/FqgEM5SNgAsTuuY7wBohSNRddfvahV1
-xPdXUrALuDH/NmIzaYZW6hh6mOhl+R7lP2XXZbFTpTGVdoosdBTGkjbPGKMrT/L8
-hwLvFezXaHZzqj4hLnmqFbhu+dDH55EE1HT5RP7kxGCq1AMuwlsjOVxURS0FZi87
-Oaq19NKsyWfdf8igONsk0GBt5HeG+93fJkW/SxssTJdz1xc91KgGDlP3nAW3xBAz
-TRvgiKIeMzOh+SWkTyz/cJugyxD+wXaAEL7VYsgOwilV+rbWKTDPvnNORqrLO/md
-MHZqYWkFlld2kw8i4LYc6zXOsOWlOv0ZM7VcEs7ufBADQEiZPkDNvWlzM97oDabE
-n/htdqxnoZ3NHJ1HJnz03jKSfg==
+MIIEpzCCAo+gAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v
+YmFyMCAXDTIzMDUxMDE1NTAxOVoYDzIyOTcwMjIyMTU1MDE5WjAeMRwwGgYDVQQD
+DBNwcm94eS1sb2NhbGhvc3QtU0FOMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAzBXJhQZDR71Gn08DGuBulBNOsDDqiMo65DmSEsF3UYwNPLkmXC/c/LFa
+vw5H/wlgMHmOVSb+0KHtn22KagaF8NDclKZUoabJPlfVaX3pJcHva3fhYnbY5FSR
+QLwLEXS4MLvUAnfWvdLQ563ffZiWdEKtU7OIyNwd21FjhO5+hXMUXtTI8AFfZ1Lt
+lIf31qooiyyEmIy5kbU4mYBds9TblZYJ7x2hb4bIF4b3Ch5yO1CMU+XO1IzPzIE9
+RlX/ZSULNjExpiInR5ZZOMHNZqaag5jcuC4QjbpFrqogbuMLvezmY7VAVdT+l7Hx
+jZrAokaOo+2gG+1AsAClKPnaA73BqQIDAQABo4H5MIH2MAkGA1UdEwQCMAAwEQYJ
+YIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRl
+ZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFMUzc2cDt1EI9L3TzU/cz4MR
+U605MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATBBBgNVHSMEOjA4gBRXC+nL
+I+i/Rz5Qej9FfqEYQ50VJ6EVpBMwETEPMA0GA1UEAwwGZm9vYmFyggkA1+KHT6B5
+4gwwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3
+DQEBCwUAA4ICAQBD72cpmgxTl3z8cnNsjUh4TuzjFJ3ZHoNM1vBW6cTY3vVU+6U7
+/1kjdSZ08IaQ0E1BJQOH4GCkmzM9vRx5uNuGHDgJJg2APvkeKBENPWseGnqa+vwY
+In/9RlXCL1ZcXIpF8nR65GzQ4OrsdLcNqPPKGM+kvqDgSjLKFX5dBla3cXzg3Bn6
+vj6UhCC+ljRhC/DR1jFJC7AguPlcSQgTm0XAb1gWgQsM+GY4WIPUsLwUNY3iHdUt
+6gKuQuGIIlqwz+UxscvT6dJeiFW9YqyFqk78GGtl+Z78kycMximq8GRuctzZla44
+rmSexkSKCw8O1Gl+eeBG0HWWKhpgrzAj3NJnDQgqnVgpCR7ICNU6iC0a3EfcXb0N
+XFTxXVptDd68GGct3Rv+iw4DGbAP8llp0HpPoTN09yLv/5DhS46sEwBvAJtVg9KW
+26iByamNxqYhPRTTQ3EoxuptLZG5WL/sGHXEjBBDiGAIwLud+5CAHtWj6ueKFvf0
+18s1kwNV5MxYMR7fbuQbbq06dlbli07Zca8Rkqd64mbM0nPz7Og7Z/BqMRCC6MQe
+rsNUp+JChv5Dda3vg9ccL5GUHFedHEOUsUeybJb9g2kPbOIYm2WOcQgBs3NGqjwu
+BxTNA67cWlHaxUFTzPX8yNtOdieZmuxAaAfWEOH5aGtdUpU9AfSnQBFhCg==
 -----END CERTIFICATE-----
diff --git a/tests/certificate-authority/server-keys/proxy.csr.pem b/tests/certificate-authority/server-keys/proxy.csr.pem
index 8dbf74bb819ad..6cebd3548a14b 100644
--- a/tests/certificate-authority/server-keys/proxy.csr.pem
+++ b/tests/certificate-authority/server-keys/proxy.csr.pem
@@ -1,15 +1,15 @@
 -----BEGIN CERTIFICATE REQUEST-----
-MIICZzCCAU8CAQAwIjEgMB4GA1UEAwwXcHJveHkucHVsc2FyLmFwYWNoZS5vcmcw
-ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDz5wZ5B6ir17r+wSjqEPJw
-Gu4BFQI80QDYTsdmJN/yaTdfwLIePFJbT4x7aKdZFy654zNw2oULqJc9QPzebDdk
-RqHvv2P2iPxUQxduchsxKDP5MGQ45no0I9h2+mLLqSPmGLT6+WQ0niAb75Vbq131
-yZGX+Ql2HhI6q9aRKS6TQmaFqDoogu9/9CgaDk6EefmpMPg29kK+RUs+wLk0PSlG
-dR2TZeMWQbhIWumrxEnUcpB9l0Mw7KjAVa7gch28ztaYP6KeIjoakOgxbuu73io1
-szC8Ske8fwPFDnfX/Q9/qWmwaFO9mzaoMGygpxEaKN2NIHtQdcexjkUexh61kipV
-AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAMBYwlvpcPsZQQMwUbts7GsX35Hcn
-FAl8iWcKr9uw/9sSrZkstI9Aa8As+KYPeY3Z2p5TYY1TXokZa936NB00CWnY+gxY
-lfKXy31yPqEHSwir1pQDU+WTILwZfbptFpAFEBy0SCDWrBZJUbM1ngqcVDg9jlQi
-iZMDYbsnZ828Hn4e97P83bOubSBWIf1Rp6LcbIzJtwGCGVp+XPJYPMFXmpzAtwrT
-tSgzCnHXseYKwIbjr+ReW58jE8Z59UqBm3/VeidLg94VfITuN5et42yypWd9Z7DU
-C/qE8gjrqlvl49Xi6ye/RxKTMN+8TiQigU5ngEnYvNKbpKhU4veXHKjfrg==
+MIICYzCCAUsCAQAwHjEcMBoGA1UEAwwTcHJveHktbG9jYWxob3N0LVNBTjCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMwVyYUGQ0e9Rp9PAxrgbpQTTrAw
+6ojKOuQ5khLBd1GMDTy5Jlwv3PyxWr8OR/8JYDB5jlUm/tCh7Z9timoGhfDQ3JSm
+VKGmyT5X1Wl96SXB72t34WJ22ORUkUC8CxF0uDC71AJ31r3S0Oet332YlnRCrVOz
+iMjcHdtRY4TufoVzFF7UyPABX2dS7ZSH99aqKIsshJiMuZG1OJmAXbPU25WWCe8d
+oW+GyBeG9woecjtQjFPlztSMz8yBPUZV/2UlCzYxMaYiJ0eWWTjBzWammoOY3Lgu
+EI26Ra6qIG7jC73s5mO1QFXU/pex8Y2awKJGjqPtoBvtQLAApSj52gO9wakCAwEA
+AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQCqq+WUWY5w8RHsMi/zeR0JjXhbgdYSlsfP
+J0fUTB88Jr/litM2u96/HFPC4K8MDlei7cKccyrRsLd+iEbG3ydNRB66WBCjq93o
+/5LSMYGCejMDAdeE8qWBDf53Xlg+hGhMlFbbmL4JGTKX0OjAKnF5xT5i7n9rh/dM
+FwoIU6ac+5btszD62IRGhyOmOHXSqL+KYArKhXKzGICFKvrOwfGrSC7Rt9zwV+lL
+UB6NRWR5viSgIkwYw/W4R9M1iQPQLjGm/ibjW/FXWzr98LNgs8kjqMoAIDs7z8YU
+FCT6YDb8zJlqiOBKnU7ReetKsHwPM2mAyWMS6z8R3LOSNdgrP70Y
 -----END CERTIFICATE REQUEST-----
diff --git a/tests/certificate-authority/server-keys/proxy.key-pk8.pem b/tests/certificate-authority/server-keys/proxy.key-pk8.pem
index 114fe2fb04d09..0dc72cde403f8 100644
--- a/tests/certificate-authority/server-keys/proxy.key-pk8.pem
+++ b/tests/certificate-authority/server-keys/proxy.key-pk8.pem
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDz5wZ5B6ir17r+
-wSjqEPJwGu4BFQI80QDYTsdmJN/yaTdfwLIePFJbT4x7aKdZFy654zNw2oULqJc9
-QPzebDdkRqHvv2P2iPxUQxduchsxKDP5MGQ45no0I9h2+mLLqSPmGLT6+WQ0niAb
-75Vbq131yZGX+Ql2HhI6q9aRKS6TQmaFqDoogu9/9CgaDk6EefmpMPg29kK+RUs+
-wLk0PSlGdR2TZeMWQbhIWumrxEnUcpB9l0Mw7KjAVa7gch28ztaYP6KeIjoakOgx
-buu73io1szC8Ske8fwPFDnfX/Q9/qWmwaFO9mzaoMGygpxEaKN2NIHtQdcexjkUe
-xh61kipVAgMBAAECggEBAJ/DuDC1fJ477OiNPLC+MyCN81NQIKwXt/b4+5KEGxHe
-LACT59j4aHYZkIsSDXTFQ71N/1cwPLBbWd4s4LcNqecMgWzbMK7AIpFLdWDKa9dy
-X0EemrfO+UOIK3YcI3UGsVY63un7TNFOtve1o19tzFmBFNa4saLmpcg64Y0qrbCV
-KcHslT1T07szp5s+weiMxgsD17foNSBEXLxP7+1F9NPlWuiHh+Rl2/t+K2tjrXeI
-EN9dtv29q4v9jCRU4yhIunAjLEvrMYCSGhXEGa+MRkgXkTPhhVN5nWX6M0uDyKgK
-aJJBv+/H6QVj4XetubYdLjII0L2q/vckoD5JsaYfz40CgYEA/7ID5OWbp/OOCjK1
-wbMByKwLUL5tHapZIoYdNg/w6zjjYl1TM9e18p1llOb+oPTEk+p8LigkkkDvPrEZ
-zAhAU3Z3nRWGkVOLNYycuSed283Up0Kml08vsRNGDa78bma4GaWnJpOuPx5fB1HN
-njjq9XhYzIEAHO4dT2dAQB003JMCgYEA9DFp0FnfsZsuAMLwBJJ08yHn4CjoYpMq
-TAg3JScEjnm1ELJBvqLYRHzqHVeSKUHTtVDwaAqMe43qEnQ3IuFS+dhJGfOX41Cf
-Yw7WDZvIeuPZER7WXUY27wmjGbjx6SdIuDYnYYA0P3RSGm3VcGZqaLoW7MvfDB0y
-pYpVSV6pFncCgYEAz5/dSaCoJFjAncdPj1mruSb6iTYXpF8OwdnlHmETX+1xtg3R
-4ebm93qXYbGwUUJv3SwqadBu4dOYcW+dYu/QS/WGaydvfdI41+K14CMrK7CXXLni
-TDsgnsjnuXS9xWfjVfANKmYAt4AR6f+i1zegknKGqIiXbuZrJm7Q3T7aDcECgYEA
-7tXBm6G7kzemt+Hx5VblgcOgyfLYz0kG7pR+cx0FbOCHAsyGVxFpGxtd09MJxsZ2
-bXm7mNbwbgvwa5o1Ly1Y/brYTMSewxrguX8SRv8eB2wAq6kQmuwI4KT5XDgyiwr8
-Kgf1XnyJHaMEhor0XlodK08PCw2fm3aXSafSIM+v66MCgYEAiGDfCy25tcI4UpAb
-v8WjI2Y7EXE2vJQ/mqMhKzmfME8HMzBvuzhwAERJgPHh1lNOIwH1LnF4lZS7jr75
-A78lgfTj6ZKNHpr5s5+5zdllvFwQ51SczCUnZv0flb/S5Qeciqh0a//pe9FQvL3+
-3cqpvX158ljL8FYfcPQOBuIUdjw=
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMFcmFBkNHvUaf
+TwMa4G6UE06wMOqIyjrkOZISwXdRjA08uSZcL9z8sVq/Dkf/CWAweY5VJv7Qoe2f
+bYpqBoXw0NyUplShpsk+V9Vpfeklwe9rd+FidtjkVJFAvAsRdLgwu9QCd9a90tDn
+rd99mJZ0Qq1Ts4jI3B3bUWOE7n6FcxRe1MjwAV9nUu2Uh/fWqiiLLISYjLmRtTiZ
+gF2z1NuVlgnvHaFvhsgXhvcKHnI7UIxT5c7UjM/MgT1GVf9lJQs2MTGmIidHllk4
+wc1mppqDmNy4LhCNukWuqiBu4wu97OZjtUBV1P6XsfGNmsCiRo6j7aAb7UCwAKUo
++doDvcGpAgMBAAECggEAPTAXEFQVXe/ouaDV3HwHi0vSns67srF3QK/mFMt+e6uS
+2G7mimMrTXPbMkcU3OkxtrbrLqqXYXP7K36LLkiwZcgpKkRIQYMg+RkaehtvCIwB
+vWXe5Eefta2JMzBt3RjylGHsKaVGc/k9+whNZnmWOls3Xk4Ip7gfF39qaBOdSWLy
+gVJJ9qbM6nHejl3vdSew5nRGlqGu0qonB+OhbM2GGSLEjH4BBbDlvI1Gl/mdKMLK
+XMJf6WvO/d43+y+/EWmln2NF5lXHRwD+Gk84316mf+ANFy8V3yvp7BC9kDWqLeDH
+A6wK9fmwPzwEWZKgzurN/Euz8gv1hEVcxH54HUBgVQKBgQD/iprN8phLUeA1VwJg
+zm/AX6jM4GIEig0rXkYOOz1KI7wx6DunfnFINxSpDXf93cK7k6wtmLkxu0ONO9LE
+1q4bmaBPkObfGOzBFixSOEE9ecyH9+EiFQi4O1zBqZLPObIY7tJ5xVsAmvQmwrf9
+Agy6fKNGM1SFBWyHgmTUvtLrlwKBgQDMc4slLWyTZYZn7bpbuXslRBWSp3KROP8M
+R5wCGe0xmOz2A/UEfGMruAS3/2+GJQlMpW25ACi3GX+Wq0OAlVxHZfArw9vaJSXm
+Nj1rC0bkICKYHojS0kFckTHBGUEKOAaSdP1Ehm+eUaH0kVapfw9bSQ0+Yd5Z0XHS
+65QqcY9kvwKBgFD8emc+tSlZv3boJmbLxfrv1i1oB2hs4BOYgxdLivcOMDyY3x8M
+IZbDbhbNn/Oi7m5INM8WkcrDEHuYNAoSB4fTvky5HZIi8hWXk2BTV8nF6h5FXuJQ
+TD0nAxSVS2PFYz4noijZdSfR9AK8v1a96Y7IpW5AIk8uEuE3YAFUoL/tAoGAMGXh
+uIlKPJI6APw7s17zEd1OJgtRiaMubR++hJjSl30WCx7gr5EqgLztEQl8wwqdavF2
+SecJvF5i363nKtcwow40joes0bUdhaOtYlunCnW4+r2vsghnxJvyZT2vMdYVaDId
+ik0wuw+kARsuoq0bW4athejxE94Kzd1Kk8mSIk0CgYEAiBYxMU8c821HwRzHGUi4
+MEqrOV7D0WrQIeeunT9yBZvP1/OpH9VcHXM53JOQLX/1bF3gGzLevOxvzhSid9Kg
+7avc497uam9LfmIueIsc90yH6Qf83O2pwLc6x/Jx9cyVX4elMC9b+CpXPzC4RDla
+NGH4KgEBAySS0x7x6a31/yg=
 -----END PRIVATE KEY-----
diff --git a/tests/certificate-authority/server-keys/proxy.key.pem b/tests/certificate-authority/server-keys/proxy.key.pem
index ec79e9ddf234c..17c431ba9f5ca 100644
--- a/tests/certificate-authority/server-keys/proxy.key.pem
+++ b/tests/certificate-authority/server-keys/proxy.key.pem
@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpgIBAAKCAQEA8+cGeQeoq9e6/sEo6hDycBruARUCPNEA2E7HZiTf8mk3X8Cy
-HjxSW0+Me2inWRcuueMzcNqFC6iXPUD83mw3ZEah779j9oj8VEMXbnIbMSgz+TBk
-OOZ6NCPYdvpiy6kj5hi0+vlkNJ4gG++VW6td9cmRl/kJdh4SOqvWkSkuk0Jmhag6
-KILvf/QoGg5OhHn5qTD4NvZCvkVLPsC5ND0pRnUdk2XjFkG4SFrpq8RJ1HKQfZdD
-MOyowFWu4HIdvM7WmD+iniI6GpDoMW7ru94qNbMwvEpHvH8DxQ531/0Pf6lpsGhT
-vZs2qDBsoKcRGijdjSB7UHXHsY5FHsYetZIqVQIDAQABAoIBAQCfw7gwtXyeO+zo
-jTywvjMgjfNTUCCsF7f2+PuShBsR3iwAk+fY+Gh2GZCLEg10xUO9Tf9XMDywW1ne
-LOC3DannDIFs2zCuwCKRS3VgymvXcl9BHpq3zvlDiCt2HCN1BrFWOt7p+0zRTrb3
-taNfbcxZgRTWuLGi5qXIOuGNKq2wlSnB7JU9U9O7M6ebPsHojMYLA9e36DUgRFy8
-T+/tRfTT5Vroh4fkZdv7fitrY613iBDfXbb9vauL/YwkVOMoSLpwIyxL6zGAkhoV
-xBmvjEZIF5Ez4YVTeZ1l+jNLg8ioCmiSQb/vx+kFY+F3rbm2HS4yCNC9qv73JKA+
-SbGmH8+NAoGBAP+yA+Tlm6fzjgoytcGzAcisC1C+bR2qWSKGHTYP8Os442JdUzPX
-tfKdZZTm/qD0xJPqfC4oJJJA7z6xGcwIQFN2d50VhpFTizWMnLknndvN1KdCppdP
-L7ETRg2u/G5muBmlpyaTrj8eXwdRzZ446vV4WMyBABzuHU9nQEAdNNyTAoGBAPQx
-adBZ37GbLgDC8ASSdPMh5+Ao6GKTKkwINyUnBI55tRCyQb6i2ER86h1XkilB07VQ
-8GgKjHuN6hJ0NyLhUvnYSRnzl+NQn2MO1g2byHrj2REe1l1GNu8Joxm48eknSLg2
-J2GAND90Uhpt1XBmami6FuzL3wwdMqWKVUleqRZ3AoGBAM+f3UmgqCRYwJ3HT49Z
-q7km+ok2F6RfDsHZ5R5hE1/tcbYN0eHm5vd6l2GxsFFCb90sKmnQbuHTmHFvnWLv
-0Ev1hmsnb33SONfiteAjKyuwl1y54kw7IJ7I57l0vcVn41XwDSpmALeAEen/otc3
-oJJyhqiIl27mayZu0N0+2g3BAoGBAO7VwZuhu5M3prfh8eVW5YHDoMny2M9JBu6U
-fnMdBWzghwLMhlcRaRsbXdPTCcbGdm15u5jW8G4L8GuaNS8tWP262EzEnsMa4Ll/
-Ekb/HgdsAKupEJrsCOCk+Vw4MosK/CoH9V58iR2jBIaK9F5aHStPDwsNn5t2l0mn
-0iDPr+ujAoGBAIhg3wstubXCOFKQG7/FoyNmOxFxNryUP5qjISs5nzBPBzMwb7s4
-cABESYDx4dZTTiMB9S5xeJWUu46++QO/JYH04+mSjR6a+bOfuc3ZZbxcEOdUnMwl
-J2b9H5W/0uUHnIqodGv/6XvRULy9/t3Kqb19efJYy/BWH3D0DgbiFHY8
+MIIEowIBAAKCAQEAzBXJhQZDR71Gn08DGuBulBNOsDDqiMo65DmSEsF3UYwNPLkm
+XC/c/LFavw5H/wlgMHmOVSb+0KHtn22KagaF8NDclKZUoabJPlfVaX3pJcHva3fh
+YnbY5FSRQLwLEXS4MLvUAnfWvdLQ563ffZiWdEKtU7OIyNwd21FjhO5+hXMUXtTI
+8AFfZ1LtlIf31qooiyyEmIy5kbU4mYBds9TblZYJ7x2hb4bIF4b3Ch5yO1CMU+XO
+1IzPzIE9RlX/ZSULNjExpiInR5ZZOMHNZqaag5jcuC4QjbpFrqogbuMLvezmY7VA
+VdT+l7HxjZrAokaOo+2gG+1AsAClKPnaA73BqQIDAQABAoIBAD0wFxBUFV3v6Lmg
+1dx8B4tL0p7Ou7Kxd0Cv5hTLfnurkthu5opjK01z2zJHFNzpMba26y6ql2Fz+yt+
+iy5IsGXIKSpESEGDIPkZGnobbwiMAb1l3uRHn7WtiTMwbd0Y8pRh7CmlRnP5PfsI
+TWZ5ljpbN15OCKe4Hxd/amgTnUli8oFSSfamzOpx3o5d73UnsOZ0RpahrtKqJwfj
+oWzNhhkixIx+AQWw5byNRpf5nSjCylzCX+lrzv3eN/svvxFppZ9jReZVx0cA/hpP
+ON9epn/gDRcvFd8r6ewQvZA1qi3gxwOsCvX5sD88BFmSoM7qzfxLs/IL9YRFXMR+
+eB1AYFUCgYEA/4qazfKYS1HgNVcCYM5vwF+ozOBiBIoNK15GDjs9SiO8Meg7p35x
+SDcUqQ13/d3Cu5OsLZi5MbtDjTvSxNauG5mgT5Dm3xjswRYsUjhBPXnMh/fhIhUI
+uDtcwamSzzmyGO7SecVbAJr0JsK3/QIMunyjRjNUhQVsh4Jk1L7S65cCgYEAzHOL
+JS1sk2WGZ+26W7l7JUQVkqdykTj/DEecAhntMZjs9gP1BHxjK7gEt/9vhiUJTKVt
+uQAotxl/lqtDgJVcR2XwK8Pb2iUl5jY9awtG5CAimB6I0tJBXJExwRlBCjgGknT9
+RIZvnlGh9JFWqX8PW0kNPmHeWdFx0uuUKnGPZL8CgYBQ/HpnPrUpWb926CZmy8X6
+79YtaAdobOATmIMXS4r3DjA8mN8fDCGWw24WzZ/zou5uSDTPFpHKwxB7mDQKEgeH
+075MuR2SIvIVl5NgU1fJxeoeRV7iUEw9JwMUlUtjxWM+J6Io2XUn0fQCvL9WvemO
+yKVuQCJPLhLhN2ABVKC/7QKBgDBl4biJSjySOgD8O7Ne8xHdTiYLUYmjLm0fvoSY
+0pd9Fgse4K+RKoC87REJfMMKnWrxdknnCbxeYt+t5yrXMKMONI6HrNG1HYWjrWJb
+pwp1uPq9r7IIZ8Sb8mU9rzHWFWgyHYpNMLsPpAEbLqKtG1uGrYXo8RPeCs3dSpPJ
+kiJNAoGBAIgWMTFPHPNtR8EcxxlIuDBKqzlew9Fq0CHnrp0/cgWbz9fzqR/VXB1z
+OdyTkC1/9Wxd4Bsy3rzsb84UonfSoO2r3OPe7mpvS35iLniLHPdMh+kH/NztqcC3
+OsfycfXMlV+HpTAvW/gqVz8wuEQ5WjRh+CoBAQMkktMe8emt9f8o
 -----END RSA PRIVATE KEY-----

From d92010c00e30199c5cd53021e20ba5d7bb36701c Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 17 May 2023 11:28:32 -0500
Subject: [PATCH 286/404] [feat] OIDC: support JWKS refresh for missing Key ID
 (#20338)

### Motivation

When the `AuthenticationProviderOpenID` encounters an unknown Key ID (known as the `kid` in the JWT) for a trusted token issuer, the token is rejected with the following error `java.lang.IllegalArgumentException: No JWK found for Key ID `. This behavior is technically valid, but it isn't ideal because it is possible that the Identity Provider has issued new signing keys and started using them before the Pulsar authentication provider has refreshed its cache. This PR introduces a behavior to retrieve

This PR adds a configuration called `openIDKeyIdCacheMissRefreshSeconds` that represents the length of time that must pass before the provider will reload the JWKS from the Identity Provider. The `openIDKeyIdCacheMissRefreshSeconds` setting limits the impact of an attacker invalidating the JWKS cache.

When `openIDKeyIdCacheMissRefreshSeconds <= 0`, the JWKS will be refreshed for any missing key id when the issuer is trusted. This is only meant for testing.

### Modifications

* Add `openIDKeyIdCacheMissRefreshSeconds` setting and default it to 5 minutes.
* Add functionality to invalidate and refresh the cache when a token has an unknown `kid` for a trusted issuer.

### Verifying this change

New tests are added.

### Does this pull request potentially affect one of the following parts:

This adds a new configuration, but it is very minor.

### Documentation

- [x] `doc-required`

### Matching PR in forked repository

PR in forked repository: Skipping since tests passed already on my local machine
---
 .../oidc/AuthenticationProviderOpenID.java    |   2 +
 .../broker/authentication/oidc/JwksCache.java |  48 +++++++--
 ...ticationProviderOpenIDIntegrationTest.java | 100 ++++++++++++++++++
 3 files changed, 144 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
index 00ec09bd1817f..2078666a08dd9 100644
--- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java
@@ -133,6 +133,8 @@ public class AuthenticationProviderOpenID implements AuthenticationProvider {
     static final int CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT = 18 * 60 * 60;
     static final String CACHE_EXPIRATION_SECONDS = "openIDCacheExpirationSeconds";
     static final int CACHE_EXPIRATION_SECONDS_DEFAULT = 24 * 60 * 60;
+    static final String KEY_ID_CACHE_MISS_REFRESH_SECONDS = "openIDKeyIdCacheMissRefreshSeconds";
+    static final int KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT = 5 * 60;
     static final String HTTP_CONNECTION_TIMEOUT_MILLIS = "openIDHttpConnectionTimeoutMillis";
     static final int HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT = 10_000;
     static final String HTTP_READ_TIMEOUT_MILLIS = "openIDHttpReadTimeoutMillis";
diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
index b5e038342c2e9..73934e9c1e05e 100644
--- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
+++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java
@@ -24,6 +24,8 @@
 import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT;
 import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE;
 import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS;
+import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT;
 import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric;
 import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt;
 import com.auth0.jwk.Jwk;
@@ -43,6 +45,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import javax.naming.AuthenticationException;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -52,7 +55,8 @@ public class JwksCache {
 
     // Map from an issuer's JWKS URI to its JWKS. When the Issuer is not empty, use the fallback client.
     private final AsyncLoadingCache, List> cache;
-
+    private final ConcurrentHashMap, Long> jwksLastRefreshTime = new ConcurrentHashMap<>();
+    private final long keyIdCacheMissRefreshNanos;
     private final ObjectReader reader = new ObjectMapper().readerFor(HashMap.class);
     private final AsyncHttpClient httpClient;
     private final OpenidApi openidApi;
@@ -61,7 +65,8 @@ public class JwksCache {
         // Store the clients
         this.httpClient = httpClient;
         this.openidApi = apiClient != null ? new OpenidApi(apiClient) : null;
-
+        keyIdCacheMissRefreshNanos = TimeUnit.SECONDS.toNanos(getConfigValueAsInt(config,
+                KEY_ID_CACHE_MISS_REFRESH_SECONDS, KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT));
         // Configure the cache
         int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT);
         int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS,
@@ -69,6 +74,8 @@ public class JwksCache {
         int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS,
                 CACHE_EXPIRATION_SECONDS_DEFAULT);
         AsyncCacheLoader, List> loader = (jwksUri, executor) -> {
+            // Store the time of the retrieval, even though it might be a little early or the call might fail.
+            jwksLastRefreshTime.put(jwksUri, System.nanoTime());
             if (jwksUri.isPresent()) {
                 return getJwksFromJwksUri(jwksUri.get());
             } else {
@@ -87,7 +94,37 @@ CompletableFuture getJwk(String jwksUri, String keyId) {
             incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY);
             return CompletableFuture.failedFuture(new IllegalArgumentException("jwksUri must not be null."));
         }
-        return cache.get(Optional.of(jwksUri)).thenApply(jwks -> getJwkForKID(jwks, keyId));
+        return getJwkAndMaybeReload(Optional.of(jwksUri), keyId, false);
+    }
+
+    /**
+     * Retrieve the JWK for the given key ID from the given JWKS URI. If the key ID is not found, and failOnMissingKeyId
+     * is false, then the JWK will be reloaded from the JWKS URI and the key ID will be searched for again.
+     */
+    private CompletableFuture getJwkAndMaybeReload(Optional maybeJwksUri,
+                                                        String keyId,
+                                                        boolean failOnMissingKeyId) {
+        return cache
+                .get(maybeJwksUri)
+                .thenCompose(jwks -> {
+                    try {
+                        return CompletableFuture.completedFuture(getJwkForKID(maybeJwksUri, jwks, keyId));
+                    } catch (IllegalArgumentException e) {
+                        if (failOnMissingKeyId) {
+                            throw e;
+                        } else {
+                            Long lastRefresh = jwksLastRefreshTime.get(maybeJwksUri);
+                            if (lastRefresh == null || System.nanoTime() - lastRefresh > keyIdCacheMissRefreshNanos) {
+                                // In this case, the key ID was not found, but we haven't refreshed the JWKS in a while,
+                                // so it is possible the key ID was added. Refresh the JWKS and try again.
+                                cache.synchronous().invalidate(maybeJwksUri);
+                            }
+                            // There is a small race condition where the JWKS could be refreshed by another thread,
+                            // so we retry getting the JWK, even though we might not have invalidated the cache.
+                            return getJwkAndMaybeReload(maybeJwksUri, keyId, true);
+                        }
+                    }
+                });
     }
 
     private CompletableFuture> getJwksFromJwksUri(String jwksUri) {
@@ -119,8 +156,7 @@ CompletableFuture getJwkFromKubernetesApiServer(String keyId) {
             return CompletableFuture.failedFuture(new AuthenticationException(
                     "Failed to retrieve public key from Kubernetes API server: Kubernetes fallback is not enabled."));
         }
-        return cache.get(Optional.empty(), (__, executor) -> getJwksFromKubernetesApiServer())
-                .thenApply(jwks -> getJwkForKID(jwks, keyId));
+        return getJwkAndMaybeReload(Optional.empty(), keyId, false);
     }
 
     private CompletableFuture> getJwksFromKubernetesApiServer() {
@@ -170,7 +206,7 @@ public void onDownloadProgress(long bytesRead, long contentLength, boolean done)
         return future;
     }
 
-    private Jwk getJwkForKID(List jwks, String keyId) {
+    private Jwk getJwkForKID(Optional maybeJwksUri, List jwks, String keyId) {
         for (Jwk jwk : jwks) {
             if (jwk.getId().equals(keyId)) {
                 return jwk;
diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
index 0075d70f599d3..d2d2de1a1149d 100644
--- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
+++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java
@@ -29,6 +29,7 @@
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.stubbing.Scenario;
 import io.jsonwebtoken.SignatureAlgorithm;
 import io.jsonwebtoken.impl.DefaultJwtBuilder;
 import io.jsonwebtoken.io.Decoders;
@@ -58,6 +59,7 @@
 import org.apache.pulsar.common.api.AuthData;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 /**
@@ -75,6 +77,7 @@ public class AuthenticationProviderOpenIDIntegrationTest {
     // The valid issuer
     String issuer;
     String issuerWithTrailingSlash;
+    String issuerWithMissingKid;
     // This issuer is configured to return an issuer in the openid-configuration
     // that does not match the issuer on the token
     String issuerThatFails;
@@ -90,6 +93,7 @@ void beforeClass() throws IOException {
         server.start();
         issuer = server.baseUrl();
         issuerWithTrailingSlash = issuer + "/trailing-slash/";
+        issuerWithMissingKid = issuer + "/missing-kid";
         issuerThatFails = issuer + "/fail";
         issuerK8s = issuer + "/k8s";
 
@@ -183,6 +187,50 @@ void beforeClass() throws IOException {
                                         }
                                         """.formatted(validJwk, n, e, invalidJwk))));
 
+        server.stubFor(
+                get(urlEqualTo("/missing-kid/.well-known/openid-configuration"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "%s",
+                                          "jwks_uri": "%s/keys"
+                                        }
+                                        """.formatted(issuerWithMissingKid, issuerWithMissingKid))));
+
+        // Set up JWKS endpoint where it first responds without the KID, then with the KID. This is a stateful stub.
+        // Note that the state machine is circular to make it easier to verify the two code paths that rely on
+        // this logic.
+        server.stubFor(
+                get(urlMatching( "/missing-kid/keys"))
+                        .inScenario("Changing KIDs")
+                        .whenScenarioStateIs(Scenario.STARTED)
+                        .willSetStateTo("serve-kid")
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("{\"keys\":[]}")));
+        server.stubFor(
+                get(urlMatching( "/missing-kid/keys"))
+                        .inScenario("Changing KIDs")
+                        .whenScenarioStateIs("serve-kid")
+                        .willSetStateTo(Scenario.STARTED)
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody(
+                                        """
+                                        {
+                                            "keys" : [
+                                                {
+                                                "kid":"%s",
+                                                "kty":"RSA",
+                                                "alg":"RS256",
+                                                "n":"%s",
+                                                "e":"%s"
+                                                }
+                                            ]
+                                        }
+                                        """.formatted(validJwk, n, e))));
+
         ServiceConfiguration conf = new ServiceConfiguration();
         conf.setAuthenticationEnabled(true);
         conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
@@ -207,6 +255,12 @@ void afterClass() {
         server.stop();
     }
 
+    @BeforeMethod
+    public void beforeMethod() {
+        // Scenarios are stateful. Start each test with the correct state.
+        server.resetScenarios();
+    }
+
     @Test
     public void testTokenWithValidJWK() throws Exception {
         String role = "superuser";
@@ -268,6 +322,52 @@ public void testTokenWithInvalidIssuer() throws Exception {
             assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause());
         }
     }
+    @Test
+    public void testKidCacheMissWhenRefreshConfigZero() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        // Allows us to retrieve the JWK immediately after the cache miss of the KID
+        props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "0");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid);
+
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
+    }
+
+    @Test
+    public void testKidCacheMissWhenRefreshConfigLongerThanDelta() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthenticationEnabled(true);
+        conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName()));
+        Properties props = conf.getProperties();
+        props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false");
+        // This value is high enough that the provider will not refresh the JWK
+        props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "100");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience");
+        props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid);
+
+        AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID();
+        provider.initialize(conf);
+
+        String role = "superuser";
+        String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L);
+        try {
+            provider.authenticateAsync(new AuthenticationDataCommand(token)).get();
+            fail("Expected exception");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof IllegalArgumentException, "Found exception: " + e.getCause());
+            assertTrue(e.getCause().getMessage().contains("No JWK found for Key ID valid"),
+                    "Found exception: " + e.getCause());
+        }
+    }
 
     @Test
     public void testKubernetesApiServerAsDiscoverTrustedIssuerSuccess() throws Exception {

From 09e646f72eb640987ca9162988b6b146c485cd31 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 17 May 2023 16:57:16 -0500
Subject: [PATCH 287/404] [cleanup] Remove Streaming Dispatcher Code (#20279)

Discussed on mailing list here: https://lists.apache.org/thread/ky2bkzlz93njx3ntnvkpd0l77qzzgcmv

Fixes: https://github.com/apache/pulsar/issues/19088
Fixes: https://github.com/apache/pulsar/issues/16450
Fixes: https://github.com/apache/pulsar/issues/15422
Fixes: https://github.com/apache/pulsar/issues/11428

### Motivation

I am putting this PR together to move the mailing list conversation along.

Copied from @eolivelli on the mailing list thread:

> There are many flaky tests about that feature and I believe that it
has never been used in Production by anyone, because it happened a few
times that we did some changes in the regular Dispatcher and
introduced bugs on the StreamingDispacther (usually manifested as
flaky tests)

### Modifications

* Remove all `StreamingDispatcher` code, tests, and configuration.

### Verifying this change

It should be sufficient to see the tests pass.

### Does this pull request potentially affect one of the following parts:

This affects configuration.

### Documentation

- [x] `doc-not-needed`

There are no references to "streaming dispatcher" or "streamingdispatcher" in our github `pulsar-site` repo, which indicates to me that no docs need to be updated.

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/46
---
 .../pulsar/broker/ServiceConfiguration.java   |   7 -
 ...tStreamingDispatcherMultipleConsumers.java | 240 ----------
 ...reamingDispatcherSingleActiveConsumer.java | 233 ---------
 .../persistent/PersistentSubscription.java    |  17 +-
 .../PendingReadEntryRequest.java              |  76 ---
 .../StreamingDispatcher.java                  |  53 ---
 .../StreamingEntryReader.java                 | 342 --------------
 .../streamingdispatch/package-info.java       |  19 -
 ...iloverConsumerStreamingDispatcherTest.java |  37 --
 ...entFailoverStreamingDispatcherE2ETest.java |  38 --
 ...tStreamingDispatcherBlockConsumerTest.java |  38 --
 ...atchStreamingDispatcherThrottlingTest.java |  39 --
 ...istentTopicStreamingDispatcherE2ETest.java |  42 --
 ...ersistentTopicStreamingDispatcherTest.java |  41 --
 ...erConsumerTestStreamingDispatcherTest.java |  36 --
 .../StreamingEntryReaderTests.java            | 441 ------------------
 .../api/MessageDispatchThrottlingTest.java    |   1 -
 17 files changed, 3 insertions(+), 1697 deletions(-)
 delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java
 delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java
 delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java
 delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java
 delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java
 delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java
 delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 8668c17c3d88d..9966912bc8eae 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1174,13 +1174,6 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     )
     private boolean allowOverrideEntryFilters = false;
 
-    @FieldContext(
-        category = CATEGORY_SERVER,
-        doc = "Whether to use streaming read dispatcher. Currently is in preview and can be changed "
-                + "in subsequent release."
-    )
-    private boolean streamingDispatch = false;
-
     @FieldContext(
         dynamic = true,
         category = CATEGORY_SERVER,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java
deleted file mode 100644
index 44e8a423344cb..0000000000000
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherMultipleConsumers.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import com.google.common.collect.Lists;
-import java.util.Set;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
-import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pulsar.broker.service.Subscription;
-import org.apache.pulsar.broker.service.streamingdispatch.PendingReadEntryRequest;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingEntryReader;
-
-/**
- * A {@link PersistentDispatcherMultipleConsumers} implemented {@link StreamingDispatcher}.
- * It'll use {@link StreamingEntryReader} to read new entries instead read as micro batch from managed ledger.
- */
-@Slf4j
-public class PersistentStreamingDispatcherMultipleConsumers extends PersistentDispatcherMultipleConsumers
-    implements StreamingDispatcher {
-
-    private int sendingTaskCounter = 0;
-    private final StreamingEntryReader streamingEntryReader = new StreamingEntryReader((ManagedCursorImpl) cursor,
-            this, topic);
-    private final Executor topicExecutor;
-
-    public PersistentStreamingDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor,
-                                                          Subscription subscription) {
-        super(topic, cursor, subscription);
-        this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topic.getName());
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public synchronized void readEntryComplete(Entry entry, PendingReadEntryRequest ctx) {
-
-        ReadType readType = (ReadType) ctx.ctx;
-        if (ctx.isLast()) {
-            readFailureBackoff.reduceToHalf();
-            if (readType == ReadType.Normal) {
-                havePendingRead = false;
-            } else {
-                havePendingReplayRead = false;
-            }
-        }
-
-        if (readBatchSize < serviceConfig.getDispatcherMaxReadBatchSize()) {
-            int newReadBatchSize = Math.min(readBatchSize * 2, serviceConfig.getDispatcherMaxReadBatchSize());
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Increasing read batch size from {} to {}", name, readBatchSize, newReadBatchSize);
-            }
-            readBatchSize = newReadBatchSize;
-        }
-
-        if (shouldRewindBeforeReadingOrReplaying && readType == ReadType.Normal) {
-            // All consumers got disconnected before the completion of the read operation
-            entry.release();
-            cursor.rewind();
-            shouldRewindBeforeReadingOrReplaying = false;
-            readMoreEntries();
-            return;
-        }
-
-        if (log.isDebugEnabled()) {
-            log.debug("[{}] Distributing a messages to {} consumers", name, consumerList.size());
-        }
-
-        cursor.seek(((ManagedLedgerImpl) cursor.getManagedLedger())
-                .getNextValidPosition((PositionImpl) entry.getPosition()));
-
-        long size = entry.getLength();
-        updatePendingBytesToDispatch(size);
-        // dispatch messages to a separate thread, but still in order for this subscription
-        // sendMessagesToConsumers is responsible for running broker-side filters
-        // that may be quite expensive
-        if (serviceConfig.isDispatcherDispatchMessagesInSubscriptionThread()) {
-            // setting sendInProgress here, because sendMessagesToConsumers will be executed
-            // in a separate thread, and we want to prevent more reads
-            acquireSendInProgress();
-            dispatchMessagesThread.execute(() -> {
-                if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), false)) {
-                    readMoreEntries();
-                } else {
-                    updatePendingBytesToDispatch(-size);
-                }
-            });
-        } else {
-            if (sendMessagesToConsumers(readType, Lists.newArrayList(entry), true)) {
-                readMoreEntriesAsync();
-            } else {
-                updatePendingBytesToDispatch(-size);
-            }
-        }
-        ctx.recycle();
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void canReadMoreEntries(boolean withBackoff) {
-        havePendingRead = false;
-        topic.getBrokerService().executor().schedule(() -> {
-            topicExecutor.execute(() -> {
-                synchronized (PersistentStreamingDispatcherMultipleConsumers.this) {
-                    if (!havePendingRead) {
-                        log.info("[{}] Scheduling read operation", name);
-                        readMoreEntries();
-                    } else {
-                        log.info("[{}] Skipping read since we have pendingRead", name);
-                    }
-                }
-            });
-        }, withBackoff
-                ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void notifyConsumersEndOfTopic() {
-        if (cursor.getNumberOfEntriesInBacklog(false) == 0) {
-            // Topic has been terminated and there are no more entries to read
-            // Notify the consumer only if all the messages were already acknowledged
-            checkAndApplyReachedEndOfTopicOrTopicMigration(consumerList);
-        }
-    }
-
-    @Override
-    protected void cancelPendingRead() {
-        if (havePendingRead && streamingEntryReader.cancelReadRequests()) {
-            havePendingRead = false;
-        }
-    }
-
-    @Override
-    protected synchronized void acquireSendInProgress() {
-        sendingTaskCounter++;
-    }
-
-    @Override
-    protected synchronized void releaseSendInProgress() {
-        sendingTaskCounter--;
-    }
-
-    @Override
-    protected synchronized boolean isSendInProgress() {
-        return sendingTaskCounter > 0;
-    }
-
-    @Override
-    public synchronized void readMoreEntries() {
-        if (isSendInProgress()) {
-            // we cannot read more entries while sending the previous batch
-            // otherwise we could re-read the same entries and send duplicates
-            return;
-        }
-        // totalAvailablePermits may be updated by other threads
-        int currentTotalAvailablePermits = totalAvailablePermits;
-        if (currentTotalAvailablePermits > 0 && isAtleastOneConsumerAvailable()) {
-            Pair calculateResult = calculateToRead(currentTotalAvailablePermits);
-            int messagesToRead = calculateResult.getLeft();
-            long bytesToRead = calculateResult.getRight();
-            if (-1 == messagesToRead || bytesToRead == -1) {
-                // Skip read as topic/dispatcher has exceed the dispatch rate or previous pending read hasn't complete.
-                return;
-            }
-
-            Set messagesToReplayNow = getMessagesToReplayNow(messagesToRead);
-
-            if (!messagesToReplayNow.isEmpty()) {
-                if (log.isDebugEnabled()) {
-                    log.debug("[{}] Schedule replay of {} messages for {} consumers", name, messagesToReplayNow.size(),
-                            consumerList.size());
-                }
-
-                havePendingReplayRead = true;
-                Set deletedMessages = topic.isDelayedDeliveryEnabled()
-                        ? asyncReplayEntriesInOrder(messagesToReplayNow) : asyncReplayEntries(messagesToReplayNow);
-                // clear already acked positions from replay bucket
-
-                deletedMessages.forEach(position -> redeliveryMessages.remove(((PositionImpl) position).getLedgerId(),
-                        ((PositionImpl) position).getEntryId()));
-                // if all the entries are acked-entries and cleared up from redeliveryMessages, try to read
-                // next entries as readCompletedEntries-callback was never called
-                if ((messagesToReplayNow.size() - deletedMessages.size()) == 0) {
-                    havePendingReplayRead = false;
-                    // We should not call readMoreEntries() recursively in the same thread
-                    // as there is a risk of StackOverflowError
-                    topic.getBrokerService().executor().execute(this::readMoreEntries);
-                }
-            } else if (BLOCKED_DISPATCHER_ON_UNACKMSG_UPDATER.get(this) == TRUE) {
-                log.debug("[{}] Dispatcher read is blocked due to unackMessages {} reached to max {}", name,
-                        totalUnackedMessages, topic.getMaxUnackedMessagesOnSubscription());
-            } else if (!havePendingRead) {
-                if (log.isDebugEnabled()) {
-                    log.debug("[{}] Schedule read of {} messages for {} consumers", name, messagesToRead,
-                            consumerList.size());
-                }
-                havePendingRead = true;
-                streamingEntryReader.asyncReadEntries(messagesToRead, bytesToRead,
-                        ReadType.Normal);
-            } else {
-                log.debug("[{}] Cannot schedule next read until previous one is done", name);
-            }
-        } else {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Consumer buffer is full, pause reading", name);
-            }
-        }
-    }
-
-}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java
deleted file mode 100644
index efe9de778a3e7..0000000000000
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
-import com.google.common.collect.Lists;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
-import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pulsar.broker.service.Consumer;
-import org.apache.pulsar.broker.service.EntryBatchIndexesAcks;
-import org.apache.pulsar.broker.service.EntryBatchSizes;
-import org.apache.pulsar.broker.service.SendMessageInfo;
-import org.apache.pulsar.broker.service.Subscription;
-import org.apache.pulsar.broker.service.streamingdispatch.PendingReadEntryRequest;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingEntryReader;
-import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
-
-/**
- * A {@link PersistentDispatcherSingleActiveConsumer} implemented {@link StreamingDispatcher}.
- * It'll use {@link StreamingEntryReader} to read new entries instead read as micro batch from managed ledger.
- */
-@Slf4j
-public class PersistentStreamingDispatcherSingleActiveConsumer extends PersistentDispatcherSingleActiveConsumer
-        implements StreamingDispatcher {
-
-    private final StreamingEntryReader streamingEntryReader = new StreamingEntryReader((ManagedCursorImpl) cursor,
-            this, topic);
-
-    private final Executor dispatcherExecutor;
-
-    public PersistentStreamingDispatcherSingleActiveConsumer(ManagedCursor cursor, SubType subscriptionType,
-                                                             int partitionIndex, PersistentTopic topic,
-                                                             Subscription subscription) {
-        super(cursor, subscriptionType, partitionIndex, topic, subscription);
-        this.dispatcherExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(name);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void canReadMoreEntries(boolean withBackoff) {
-        havePendingRead = false;
-        topic.getBrokerService().executor().schedule(() -> {
-             topicExecutor.execute(() -> {
-                synchronized (PersistentStreamingDispatcherSingleActiveConsumer.this) {
-                    Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
-                    if (currentConsumer != null && !havePendingRead) {
-                        if (log.isDebugEnabled()) {
-                            log.debug("[{}-{}] Scheduling read ", name, currentConsumer);
-                        }
-                        readMoreEntries(currentConsumer);
-                    } else {
-                        log.info("[{}-{}] Skipping read as we still havePendingRead {}", name,
-                                currentConsumer, havePendingRead);
-                    }
-                }
-            });
-        }, withBackoff
-                ? readFailureBackoff.next() : 0, TimeUnit.MILLISECONDS);
-    }
-
-    @Override
-    protected void cancelPendingRead() {
-        if (havePendingRead && streamingEntryReader.cancelReadRequests()) {
-            havePendingRead = false;
-        }
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public synchronized void notifyConsumersEndOfTopic() {
-        if (cursor.getNumberOfEntriesInBacklog(false) == 0) {
-            // Topic has been terminated and there are no more entries to read
-            // Notify the consumer only if all the messages were already acknowledged
-            checkAndApplyReachedEndOfTopicOrTopicMigration(consumers);
-        }
-    }
-
-    @Override
-    public String getName() {
-        return name;
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void readEntryComplete(Entry entry, PendingReadEntryRequest ctx) {
-        dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx));
-    }
-
-    public synchronized void internalReadEntryComplete(Entry entry, PendingReadEntryRequest ctx) {
-        if (ctx.isLast()) {
-            readFailureBackoff.reduceToHalf();
-            havePendingRead = false;
-        }
-
-        isFirstRead = false;
-
-        if (readBatchSize < serviceConfig.getDispatcherMaxReadBatchSize()) {
-            int newReadBatchSize = Math.min(readBatchSize * 2, serviceConfig.getDispatcherMaxReadBatchSize());
-            if (log.isDebugEnabled()) {
-                log.debug("[{}-{}] Increasing read batch size from {} to {}", name,
-                        ((Consumer) ctx.ctx).consumerName(), readBatchSize, newReadBatchSize);
-            }
-            readBatchSize = newReadBatchSize;
-        }
-
-        Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
-
-        if (isKeyHashRangeFiltered) {
-            byte[] key = peekStickyKey(entry.getDataBuffer());
-            Consumer consumer = stickyKeyConsumerSelector.select(key);
-            // Skip the entry if it's not for current active consumer.
-            if (consumer == null || currentConsumer != consumer) {
-                entry.release();
-                return;
-            }
-        }
-        Consumer consumer = (Consumer) ctx.ctx;
-        ctx.recycle();
-        if (currentConsumer == null || consumer != currentConsumer) {
-            // Active consumer has changed since the read request has been issued. We need to rewind the cursor and
-            // re-issue the read request for the new consumer
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Rewind because no available consumer found to dispatch message to.", name);
-            }
-
-            entry.release();
-            streamingEntryReader.cancelReadRequests();
-            havePendingRead = false;
-            if (currentConsumer != null) {
-                notifyActiveConsumerChanged(currentConsumer);
-                readMoreEntries(currentConsumer);
-            }
-        } else {
-            EntryBatchSizes batchSizes = EntryBatchSizes.get(1);
-            SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal();
-            EntryBatchIndexesAcks batchIndexesAcks = EntryBatchIndexesAcks.get(1);
-            filterEntriesForConsumer(Lists.newArrayList(entry), batchSizes, sendMessageInfo, batchIndexesAcks,
-                    cursor, false, consumer);
-            // Update cursor's read position.
-            cursor.seek(((ManagedLedgerImpl) cursor.getManagedLedger())
-                    .getNextValidPosition((PositionImpl) entry.getPosition()));
-            dispatchEntriesToConsumer(currentConsumer, Lists.newArrayList(entry), batchSizes,
-                    batchIndexesAcks, sendMessageInfo, DEFAULT_CONSUMER_EPOCH);
-        }
-    }
-
-    @Override
-    protected void readMoreEntries(Consumer consumer) {
-        // consumer can be null when all consumers are disconnected from broker.
-        // so skip reading more entries if currently there is no active consumer.
-        if (null == consumer) {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Skipping read for the topic, Due to the current consumer is null", topic.getName());
-            }
-            return;
-        }
-        if (havePendingRead) {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Skipping read for the topic, Due to we have pending read.", topic.getName());
-            }
-            return;
-        }
-
-        if (consumer.getAvailablePermits() > 0) {
-            synchronized (this) {
-                if (havePendingRead) {
-                    if (log.isDebugEnabled()) {
-                        log.debug("[{}] Skipping read for the topic, Due to we have pending read.", topic.getName());
-                    }
-                    return;
-                }
-
-                Pair calculateResult = calculateToRead(consumer);
-                int messagesToRead = calculateResult.getLeft();
-                long bytesToRead = calculateResult.getRight();
-
-
-                if (-1 == messagesToRead || bytesToRead == -1) {
-                    // Skip read as topic/dispatcher has exceed the dispatch rate.
-                    return;
-                }
-
-                // Schedule read
-                if (log.isDebugEnabled()) {
-                    log.debug("[{}-{}] Schedule read of {} messages", name, consumer, messagesToRead);
-                }
-                havePendingRead = true;
-
-                if (consumer.readCompacted()) {
-                    topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, isFirstRead,
-                            this, consumer);
-                } else {
-                    streamingEntryReader.asyncReadEntries(messagesToRead, bytesToRead, consumer);
-                }
-            }
-        } else {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}-{}] Consumer buffer is full, pause reading", name, consumer);
-            }
-        }
-    }
-
-}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index d283cac77c7b6..09dabcd4bfc59 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -223,26 +223,18 @@ public CompletableFuture addConsumer(Consumer consumer) {
 
                 if (dispatcher == null || !dispatcher.isConsumerConnected()) {
                     Dispatcher previousDispatcher = null;
-                    boolean useStreamingDispatcher = topic.getBrokerService().getPulsar()
-                            .getConfiguration().isStreamingDispatch();
                     switch (consumer.subType()) {
                         case Exclusive:
                             if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) {
                                 previousDispatcher = dispatcher;
-                                dispatcher = useStreamingDispatcher
-                                        ? new PersistentStreamingDispatcherSingleActiveConsumer(
-                                        cursor, SubType.Exclusive, 0, topic, this)
-                                        : new PersistentDispatcherSingleActiveConsumer(
+                                dispatcher = new PersistentDispatcherSingleActiveConsumer(
                                         cursor, SubType.Exclusive, 0, topic, this);
                             }
                             break;
                         case Shared:
                             if (dispatcher == null || dispatcher.getType() != SubType.Shared) {
                                 previousDispatcher = dispatcher;
-                                dispatcher = useStreamingDispatcher
-                                        ? new PersistentStreamingDispatcherMultipleConsumers(
-                                        topic, cursor, this)
-                                        : new PersistentDispatcherMultipleConsumers(topic, cursor, this);
+                                dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor, this);
                             }
                             break;
                         case Failover:
@@ -256,10 +248,7 @@ public CompletableFuture addConsumer(Consumer consumer) {
 
                             if (dispatcher == null || dispatcher.getType() != SubType.Failover) {
                                 previousDispatcher = dispatcher;
-                                dispatcher = useStreamingDispatcher
-                                        ? new PersistentStreamingDispatcherSingleActiveConsumer(
-                                        cursor, SubType.Failover, partitionIndex, topic, this) :
-                                        new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover,
+                                dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover,
                                                 partitionIndex, topic, this);
                             }
                             break;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java
deleted file mode 100644
index e98e5de07c165..0000000000000
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/PendingReadEntryRequest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.streamingdispatch;
-
-import io.netty.util.Recycler;
-import lombok.Data;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
-
-/**
- * Representing a pending read request to read an entry from {@link ManagedLedger} carrying necessary context.
- */
-@Data
-public class PendingReadEntryRequest {
-
-    private static final Recycler RECYCLER = new Recycler() {
-        protected PendingReadEntryRequest newObject(Recycler.Handle handle) {
-            return new PendingReadEntryRequest(handle);
-        }
-    };
-
-    public static PendingReadEntryRequest create(Object ctx, PositionImpl position) {
-        PendingReadEntryRequest pendingReadEntryRequest = RECYCLER.get();
-        pendingReadEntryRequest.ctx = ctx;
-        pendingReadEntryRequest.position = position;
-        pendingReadEntryRequest.retry = 0;
-        pendingReadEntryRequest.isLast = false;
-        return pendingReadEntryRequest;
-    }
-
-    public void recycle() {
-        entry = null;
-        ctx = null;
-        position = null;
-        retry = -1;
-        recyclerHandle.recycle(this);
-    }
-
-    public boolean isLastRequest() {
-        return isLast;
-    }
-
-    private final Recycler.Handle recyclerHandle;
-
-    // Entry read from ledger
-    public Entry entry;
-
-    // Passed in context that'll be pass to callback
-    public Object ctx;
-
-    // Position of entry to be read
-    public PositionImpl position;
-
-    // Number of time request has been retried.
-    int retry;
-
-    // If request is the last one of a set of requests.
-    boolean isLast;
-}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java
deleted file mode 100644
index 4e9e8befe0f9e..0000000000000
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingDispatcher.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.streamingdispatch;
-
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.pulsar.broker.service.Dispatcher;
-import org.apache.pulsar.common.classification.InterfaceStability;
-
-/**
- * A {@link Dispatcher} that'll use {@link StreamingEntryReader} to read entries from {@link ManagedLedger}.
- */
-@InterfaceStability.Unstable
-public interface StreamingDispatcher extends Dispatcher {
-
-    /**
-     * Notify dispatcher issued read entry request has complete.
-     * @param entry Entry read.
-     * @param ctx   Context passed in when issuing read entries request.
-     */
-    void readEntryComplete(Entry entry, PendingReadEntryRequest ctx);
-
-    /**
-     * Notify dispatcher can issue next read request.
-     */
-    void canReadMoreEntries(boolean withBackoff);
-
-    /**
-     * Notify dispatcher to inform consumers reached end of topic.
-     */
-    void notifyConsumersEndOfTopic();
-
-    /**
-     * @return Name of the dispatcher.
-     */
-    String getName();
-}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java
deleted file mode 100644
index 6ffc5ba0f623a..0000000000000
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReader.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.streamingdispatch;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.WaitingEntryCallBack;
-import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
-import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException;
-import org.apache.pulsar.client.impl.Backoff;
-
-/**
- * Entry reader that fulfill read request by streamline the read instead of reading with micro batch.
- */
-@Slf4j
-public class StreamingEntryReader implements AsyncCallbacks.ReadEntryCallback, WaitingEntryCallBack {
-
-    private final int maxRetry = 3;
-
-    // Queue for read request issued yet waiting for complete from managed ledger.
-    private ConcurrentLinkedQueue issuedReads = new ConcurrentLinkedQueue<>();
-
-    // Queue for read request that's wait for new entries from managed ledger.
-    private ConcurrentLinkedQueue pendingReads = new ConcurrentLinkedQueue<>();
-
-    private final ManagedCursorImpl cursor;
-
-    private final StreamingDispatcher dispatcher;
-
-    private final PersistentTopic topic;
-
-    private final Executor topicExecutor;
-
-    private final Executor dispatcherExecutor;
-
-    private AtomicInteger currentReadSizeByte = new AtomicInteger(0);
-
-    private volatile State state;
-
-    private static final AtomicReferenceFieldUpdater STATE_UPDATER =
-            AtomicReferenceFieldUpdater.newUpdater(StreamingEntryReader.class, State.class, "state");
-
-    private volatile long maxReadSizeByte;
-
-    private final Backoff readFailureBackoff = new Backoff(10, TimeUnit.MILLISECONDS,
-            1, TimeUnit.SECONDS, 0, TimeUnit.MILLISECONDS);
-
-    public StreamingEntryReader(ManagedCursorImpl cursor, StreamingDispatcher dispatcher, PersistentTopic topic) {
-        this.cursor = cursor;
-        this.dispatcher = dispatcher;
-        this.topic = topic;
-        this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topic.getName());
-        this.dispatcherExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(dispatcher.getName());
-    }
-
-    /**
-     * Read entries in streaming way, that said instead of reading with micro batch and send entries to consumer after
-     * all entries in the batch are read from ledger, this method will fire numEntriesToRead requests to managedLedger
-     * and send entry to consumer whenever it is read and all entries before it have been sent to consumer.
-     * @param numEntriesToRead number of entry to read from ledger.
-     * @param maxReadSizeByte maximum byte will be read from ledger.
-     * @param ctx Context send along with read request.
-     */
-    public synchronized void asyncReadEntries(int numEntriesToRead, long maxReadSizeByte, Object ctx) {
-        if (STATE_UPDATER.compareAndSet(this, State.Canceling, State.Canceled)) {
-            internalCancelReadRequests();
-        }
-
-        if (!issuedReads.isEmpty() || !pendingReads.isEmpty()) {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] There's pending streaming read not completed yet. Not scheduling next read request.",
-                        cursor.getName());
-            }
-            return;
-        }
-
-        PositionImpl nextReadPosition = (PositionImpl) cursor.getReadPosition();
-        ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger();
-        // Edge case, when a old ledger is full and new ledger is not yet opened, position can point to next
-        // position of the last confirmed position, but it'll be an invalid position. So try to update the position.
-        if (!managedLedger.isValidPosition(nextReadPosition)) {
-            nextReadPosition = managedLedger.getNextValidPosition(nextReadPosition);
-        }
-        boolean hasEntriesToRead = managedLedger.hasMoreEntries(nextReadPosition);
-        currentReadSizeByte.set(0);
-        STATE_UPDATER.set(this, State.Issued);
-        this.maxReadSizeByte = maxReadSizeByte;
-        for (int c = 0; c < numEntriesToRead; c++) {
-            PendingReadEntryRequest pendingReadEntryRequest = PendingReadEntryRequest.create(ctx, nextReadPosition);
-            // Make sure once we start putting request into pending requests queue, we won't put any following request
-            // to issued requests queue in order to guarantee the order.
-            if (hasEntriesToRead && managedLedger.hasMoreEntries(nextReadPosition)) {
-                issuedReads.offer(pendingReadEntryRequest);
-            } else {
-                pendingReads.offer(pendingReadEntryRequest);
-            }
-            nextReadPosition = managedLedger.getNextValidPosition(nextReadPosition);
-        }
-
-        // Issue requests.
-        for (PendingReadEntryRequest request : issuedReads) {
-            managedLedger.asyncReadEntry(request.position, this, request);
-        }
-
-        if (!pendingReads.isEmpty()) {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}} Streaming entry reader has {} pending read requests waiting on new entry."
-                        , cursor.getName(), pendingReads.size());
-            }
-            // If new entries are available after we put request into pending queue, fire read.
-            // Else register callback with managed ledger to get notify when new entries are available.
-            if (managedLedger.hasMoreEntries(pendingReads.peek().position)) {
-                entriesAvailable();
-            } else if (managedLedger.isTerminated()) {
-                dispatcher.notifyConsumersEndOfTopic();
-                cleanQueue(pendingReads);
-                if (issuedReads.size() == 0) {
-                    dispatcher.canReadMoreEntries(true);
-                }
-            } else {
-                managedLedger.addWaitingEntryCallBack(this);
-            }
-        }
-    }
-
-    @Override
-    public void readEntryComplete(Entry entry, Object ctx) {
-        // Don't block caller thread, complete read entry with dispatcher dedicated thread.
-        dispatcherExecutor.execute(() -> internalReadEntryComplete(entry, ctx));
-    }
-
-    private void internalReadEntryComplete(Entry entry, Object ctx) {
-        PendingReadEntryRequest pendingReadEntryRequest = (PendingReadEntryRequest) ctx;
-        pendingReadEntryRequest.entry = entry;
-        readFailureBackoff.reduceToHalf();
-        Entry readEntry;
-        // If we have entry to send to dispatcher.
-        if (!issuedReads.isEmpty() && issuedReads.peek() == pendingReadEntryRequest) {
-            while (!issuedReads.isEmpty() && issuedReads.peek().entry != null) {
-                PendingReadEntryRequest firstPendingReadEntryRequest = issuedReads.poll();
-                readEntry = firstPendingReadEntryRequest.entry;
-                currentReadSizeByte.addAndGet(readEntry.getLength());
-                //Cancel remaining requests and reset cursor if maxReadSizeByte exceeded.
-                if (currentReadSizeByte.get() > maxReadSizeByte) {
-                    cancelReadRequests(readEntry.getPosition());
-                    dispatcher.canReadMoreEntries(false);
-                    STATE_UPDATER.set(this, State.Completed);
-                    return;
-                } else {
-                    // All request has been completed, mark returned entry as last.
-                    if (issuedReads.isEmpty() && pendingReads.isEmpty()) {
-                        firstPendingReadEntryRequest.isLast = true;
-                        STATE_UPDATER.set(this, State.Completed);
-                    }
-                    dispatcher.readEntryComplete(readEntry, firstPendingReadEntryRequest);
-                }
-            }
-        } else if (!issuedReads.isEmpty() && issuedReads.peek().retry > maxRetry) {
-            cancelReadRequests(issuedReads.peek().position);
-            dispatcher.canReadMoreEntries(true);
-            STATE_UPDATER.set(this, State.Completed);
-        }
-    }
-
-    @Override
-    public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
-        // Don't block caller thread, complete read entry fail with dispatcher dedicated thread.
-        dispatcherExecutor.execute(() -> internalReadEntryFailed(exception, ctx));
-    }
-
-    private void internalReadEntryFailed(ManagedLedgerException exception, Object ctx) {
-        PendingReadEntryRequest pendingReadEntryRequest = (PendingReadEntryRequest) ctx;
-        PositionImpl readPosition = pendingReadEntryRequest.position;
-        pendingReadEntryRequest.retry++;
-        long waitTimeMillis = readFailureBackoff.next();
-        if (exception.getCause() instanceof TransactionBufferException.TransactionNotSealedException
-                || exception.getCause() instanceof ManagedLedgerException.OffloadReadHandleClosedException) {
-            waitTimeMillis = 1;
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Error reading transaction entries : {}, - Retrying to read in {} seconds",
-                        cursor.getName(), exception.getMessage(), waitTimeMillis / 1000.0);
-            }
-        } else if (!(exception instanceof ManagedLedgerException.TooManyRequestsException)) {
-            log.error("[{} Error reading entries at {} : {} - Retrying to read in {} seconds", cursor.getName(),
-                    readPosition, exception.getMessage(), waitTimeMillis / 1000.0);
-        } else {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Got throttled by bookies while reading at {} : {} - Retrying to read in {} seconds",
-                        cursor.getName(), readPosition, exception.getMessage(), waitTimeMillis / 1000.0);
-            }
-        }
-        if (!issuedReads.isEmpty()) {
-            if (issuedReads.peek().retry > maxRetry) {
-                cancelReadRequests(issuedReads.peek().position);
-                dispatcher.canReadMoreEntries(true);
-                STATE_UPDATER.set(this, State.Completed);
-                return;
-            }
-            if (pendingReadEntryRequest.retry <= maxRetry) {
-                retryReadRequest(pendingReadEntryRequest, waitTimeMillis);
-            }
-        }
-    }
-
-    // Cancel all issued and pending request and update cursor's read position.
-    private void cancelReadRequests(Position position) {
-        if (!issuedReads.isEmpty()) {
-            cleanQueue(issuedReads);
-            cursor.seek(position);
-        }
-
-        if (!pendingReads.isEmpty()) {
-            cleanQueue(pendingReads);
-        }
-    }
-
-    private void internalCancelReadRequests() {
-        Position readPosition = !issuedReads.isEmpty() ? issuedReads.peek().position : pendingReads.peek().position;
-        cancelReadRequests(readPosition);
-    }
-
-    public boolean cancelReadRequests() {
-        if (STATE_UPDATER.compareAndSet(this, State.Issued, State.Canceling)) {
-            // Don't block caller thread, complete cancel read with dispatcher dedicated thread.
-             topicExecutor.execute(() -> {
-                synchronized (StreamingEntryReader.this) {
-                    if (STATE_UPDATER.compareAndSet(this, State.Canceling, State.Canceled)) {
-                        internalCancelReadRequests();
-                    }
-                }
-            });
-            return true;
-        }
-        return false;
-    }
-
-    private void cleanQueue(Queue queue) {
-        while (!queue.isEmpty()) {
-            PendingReadEntryRequest pendingReadEntryRequest = queue.poll();
-            if (pendingReadEntryRequest.entry != null) {
-                pendingReadEntryRequest.entry.release();
-                pendingReadEntryRequest.recycle();
-            }
-        }
-    }
-
-    private void retryReadRequest(PendingReadEntryRequest pendingReadEntryRequest, long delay) {
-        topic.getBrokerService().executor().schedule(() -> {
-            // Jump again into dispatcher dedicated thread
-            dispatcherExecutor.execute(() -> {
-                ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger();
-                managedLedger.asyncReadEntry(pendingReadEntryRequest.position, this, pendingReadEntryRequest);
-            });
-        }, delay, TimeUnit.MILLISECONDS);
-    }
-
-    @Override
-    public void entriesAvailable() {
-        dispatcherExecutor.execute(this::internalEntriesAvailable);
-    }
-
-    private synchronized void internalEntriesAvailable() {
-        if (log.isDebugEnabled()) {
-            log.debug("[{}} Streaming entry reader get notification of newly added entries from managed ledger,"
-                    + " trying to issued pending read requests.", cursor.getName());
-        }
-        ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) cursor.getManagedLedger();
-        List newlyIssuedRequests = new ArrayList<>();
-        if (!pendingReads.isEmpty()) {
-            // Edge case, when a old ledger is full and new ledger is not yet opened, position can point to next
-            // position of the last confirmed position, but it'll be an invalid position. So try to update the position.
-            if (!managedLedger.isValidPosition(pendingReads.peek().position)) {
-                pendingReads.peek().position = managedLedger.getNextValidPosition(pendingReads.peek().position);
-            }
-            while (!pendingReads.isEmpty() && managedLedger.hasMoreEntries(pendingReads.peek().position)) {
-                PendingReadEntryRequest next = pendingReads.poll();
-                issuedReads.offer(next);
-                newlyIssuedRequests.add(next);
-                // Need to update the position because when the PendingReadEntryRequest is created, the position could
-                // be all set to managed ledger's last confirmed position.
-                if (!pendingReads.isEmpty()) {
-                    pendingReads.peek().position = managedLedger.getNextValidPosition(next.position);
-                }
-            }
-
-            for (PendingReadEntryRequest request : newlyIssuedRequests) {
-                managedLedger.asyncReadEntry(request.position, this, request);
-            }
-
-            if (!pendingReads.isEmpty()) {
-                if (log.isDebugEnabled()) {
-                    log.debug("[{}} Streaming entry reader has {} pending read requests waiting on new entry."
-                            , cursor.getName(), pendingReads.size());
-                }
-                if (managedLedger.hasMoreEntries(pendingReads.peek().position)) {
-                    entriesAvailable();
-                } else {
-                    managedLedger.addWaitingEntryCallBack(this);
-                }
-            }
-        }
-    }
-
-    protected State getState() {
-        return STATE_UPDATER.get(this);
-    }
-
-    enum State {
-        Issued, Canceling, Canceled, Completed;
-    }
-
-}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java
deleted file mode 100644
index 4d576d9f437f3..0000000000000
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/streamingdispatch/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.streamingdispatch;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java
deleted file mode 100644
index f86fe0701dc70..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherFailoverConsumerStreamingDispatcherTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.service.PersistentDispatcherFailoverConsumerTest;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-/**
- * PersistentDispatcherFailoverConsumerTest with {@link StreamingDispatcher}
- */
-@Test(groups = "quarantine")
-public class PersistentDispatcherFailoverConsumerStreamingDispatcherTest extends PersistentDispatcherFailoverConsumerTest {
-
-    @BeforeMethod(alwaysRun = true)
-    public void setup() throws Exception {
-        super.setup();
-        pulsarTestContext.getConfig().setStreamingDispatch(true);
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java
deleted file mode 100644
index 92352cde47ff0..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentFailoverStreamingDispatcherE2ETest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.service.PersistentFailoverE2ETest;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-/**
- * PersistentFailoverE2ETest with {@link StreamingDispatcher}
- */
-@Test(groups = "broker")
-public class PersistentFailoverStreamingDispatcherE2ETest extends PersistentFailoverE2ETest {
-
-    @BeforeClass
-    @Override
-    protected void setup() throws Exception {
-        conf.setStreamingDispatch(true);
-        super.setup();
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java
deleted file mode 100644
index ef515cd85bf98..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherBlockConsumerTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.apache.pulsar.client.api.DispatcherBlockConsumerTest;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-/**
- * DispatcherBlockConsumerTest with {@link StreamingDispatcher}
- */
-@Test(groups = "flaky")
-public class PersistentStreamingDispatcherBlockConsumerTest extends DispatcherBlockConsumerTest {
-
-    @BeforeMethod
-    @Override
-    protected void setup() throws Exception {
-        super.setup();
-        conf.setStreamingDispatch(true);
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java
deleted file mode 100644
index fc62d84b58634..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.apache.pulsar.client.api.SubscriptionMessageDispatchThrottlingTest;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-/**
- * SubscriptionMessageDispatchThrottlingTest with {@link StreamingDispatcher}
- */
-@Test(groups = "flaky")
-public class PersistentSubscriptionMessageDispatchStreamingDispatcherThrottlingTest
-    extends SubscriptionMessageDispatchThrottlingTest {
-
-    @BeforeClass
-    @Override
-    protected void setup() throws Exception {
-        super.setup();
-        conf.setStreamingDispatch(true);
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java
deleted file mode 100644
index b4aa4793ba6fc..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherE2ETest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.service.PersistentTopicE2ETest;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.testng.annotations.Test;
-
-/**
- * PersistentTopicE2ETest with {@link StreamingDispatcher}
- */
-@Test(groups = "flaky")
-public class PersistentTopicStreamingDispatcherE2ETest extends PersistentTopicE2ETest {
-
-    @Override
-    protected void doInitConf() throws Exception {
-        super.doInitConf();
-        conf.setStreamingDispatch(true);
-    }
-
-    @Override
-    @Test
-    public void testMessageRedelivery() throws Exception {
-        super.testMessageRedelivery();
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java
deleted file mode 100644
index 440cbbe290c4d..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicStreamingDispatcherTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.ServiceConfiguration;
-import org.apache.pulsar.broker.service.PersistentTopicTest;
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-/**
- * PersistentTopicTest with {@link StreamingDispatcher}
- */
-@Test(groups = "broker")
-public class PersistentTopicStreamingDispatcherTest extends PersistentTopicTest {
-
-    @BeforeMethod(alwaysRun = true)
-    public void setup() throws Exception {
-        super.setup();
-        ServiceConfiguration config = pulsarTestContext.getConfig();
-        config.setTopicLevelPoliciesEnabled(false);
-        config.setSystemTopicEnabled(false);
-        config.setStreamingDispatch(true);
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java
deleted file mode 100644
index 706571eeb712f..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SimpleProducerConsumerTestStreamingDispatcherTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.persistent;
-
-import org.apache.pulsar.broker.service.streamingdispatch.StreamingDispatcher;
-import org.apache.pulsar.client.api.SimpleProducerConsumerTest;
-import org.testng.annotations.Test;
-
-/**
- * SimpleProducerConsumerTest with {@link StreamingDispatcher}
- */
-@Test(groups = "flaky")
-public class SimpleProducerConsumerTestStreamingDispatcherTest extends SimpleProducerConsumerTest {
-
-    @Override
-    protected void doInitConf() throws Exception {
-        super.doInitConf();
-        conf.setStreamingDispatch(true);
-    }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java
deleted file mode 100644
index 5eb29b8ef2870..0000000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/streamingdispatch/StreamingEntryReaderTests.java
+++ /dev/null
@@ -1,441 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.service.streamingdispatch;
-
-import static org.awaitility.Awaitility.await;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Stack;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.bookkeeper.common.util.OrderedExecutor;
-import org.apache.bookkeeper.common.util.OrderedScheduler;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.impl.EntryImpl;
-import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
-import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.apache.pulsar.broker.service.BrokerService;
-import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.testng.annotations.Test;
-
-/**
- * Tests for {@link StreamingEntryReader}
- */
-@Test(groups = "flaky")
-public class StreamingEntryReaderTests extends MockedBookKeeperTestCase {
-
-    private static final Charset Encoding = StandardCharsets.UTF_8;
-    private PersistentTopic mockTopic;
-    private StreamingDispatcher mockDispatcher;
-    private BrokerService mockBrokerService;
-    private EventLoopGroup eventLoopGroup;
-    private OrderedExecutor orderedExecutor;
-    private ManagedLedgerConfig config;
-    private ManagedLedgerImpl ledger;
-    private ManagedCursor cursor;
-
-    @Override
-    protected void setUpTestCase() throws Exception {
-        eventLoopGroup = new NioEventLoopGroup(1);
-        orderedExecutor = OrderedScheduler.newSchedulerBuilder()
-                .numThreads(1)
-                .name("StreamingEntryReaderTests").build();
-        mockTopic = mock(PersistentTopic.class);
-        mockBrokerService = mock(BrokerService.class);
-        mockDispatcher = mock(StreamingDispatcher.class);
-        config = new ManagedLedgerConfig().setMaxEntriesPerLedger(10);
-        ledger = spy((ManagedLedgerImpl) factory.open("my_test_ledger", config));
-        cursor = ledger.openCursor("test");
-        when(mockTopic.getBrokerService()).thenReturn(mockBrokerService);
-        when(mockBrokerService.executor()).thenReturn(eventLoopGroup);
-        when(mockBrokerService.getTopicOrderedExecutor()).thenReturn(orderedExecutor);
-        doAnswer(new Answer() {
-            @Override
-            public Void answer(InvocationOnMock invocationOnMock) {
-                return null;
-            }
-        }).when(mockDispatcher).notifyConsumersEndOfTopic();
-    }
-
-    @Override
-    protected void cleanUpTestCase() {
-        if (eventLoopGroup != null) {
-            eventLoopGroup.shutdownNow();
-            eventLoopGroup = null;
-        }
-        if (orderedExecutor != null) {
-            orderedExecutor.shutdownNow();
-            orderedExecutor = null;
-        }
-    }
-
-    @Test
-    public void testCanReadEntryFromMLedgerHappyPath() throws Exception {
-        AtomicInteger entryCount = new AtomicInteger(0);
-        Stack positions = new Stack<>();
-
-        for (int i = 0; i < 150; i++) {
-            ledger.addEntry(String.format("message-%d", i).getBytes(Encoding));
-        }
-
-        StreamingEntryReader streamingEntryReader =new StreamingEntryReader((ManagedCursorImpl) cursor,
-                mockDispatcher, mockTopic);
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                Entry entry = invocationOnMock.getArgument(0, Entry.class);
-                positions.push(entry.getPosition());
-                assertEquals(new String(entry.getData()), String.format("message-%d", entryCount.getAndIncrement()));
-                cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition()));
-                return null;
-            }
-        ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class));
-
-        streamingEntryReader.asyncReadEntries(50, 700, null);
-        await().until(() -> entryCount.get() == 50);
-        // Check cursor's read position has been properly updated
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        streamingEntryReader.asyncReadEntries(50, 700, null);
-        await().until(() -> entryCount.get() == 100);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        streamingEntryReader.asyncReadEntries(50, 700, null);
-        await().until(() -> entryCount.get() == 150);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-    }
-
-    @Test
-    public void testCanReadEntryFromMLedgerSizeExceededLimit() throws Exception {
-        AtomicBoolean readComplete = new AtomicBoolean(false);
-        Stack positions = new Stack<>();
-        List entries = new ArrayList<>();
-        int size = "mmmmmmmmmmessage-0".getBytes().length;
-        for (int i = 0; i < 15; i++) {
-            ledger.addEntry(String.format("mmmmmmmmmmessage-%d", i).getBytes(Encoding));
-        }
-
-        StreamingEntryReader streamingEntryReader =
-                new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic);
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                Entry entry = invocationOnMock.getArgument(0, Entry.class);
-                positions.push(entry.getPosition());
-                entries.add(new String(entry.getData()));
-                cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition()));
-                return null;
-            }
-        ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class));
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                readComplete.set(true);
-                return null;
-            }
-        ).when(mockDispatcher).canReadMoreEntries(anyBoolean());
-
-        PositionImpl position = ledger.getPositionAfterN(ledger.getFirstPosition(), 3, ManagedLedgerImpl.PositionBound.startExcluded);
-        // Make reading from mledger return out of order.
-        doAnswer(new Answer() {
-            @Override
-            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
-                AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class);
-                executor.schedule(() -> {
-                    cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(), "mmmmmmmmmmessage-2".getBytes()),
-                            invocationOnMock.getArgument(2));
-                }, 200, TimeUnit.MILLISECONDS);
-                return null;
-            }
-        }).when(ledger).asyncReadEntry(eq(position), any(), any());
-
-        // Only 2 entries should be read with this request.
-        streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null);
-        await().until(() -> readComplete.get());
-        assertEquals(entries.size(), 2);
-        // Assert cursor's read position has been properly updated to the third entry, since we should only read
-        // 2 retries with previous request
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        reset(ledger);
-        readComplete.set(false);
-        streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null);
-        await().until(() -> readComplete.get());
-        readComplete.set(false);
-        streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null);
-        await().until(() -> readComplete.get());
-        readComplete.set(false);
-        streamingEntryReader.asyncReadEntries(6, size * 2 + 1, null);
-        await().until(() -> readComplete.get());
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        assertEquals(entries.size(), 8);
-        for (int i = 0; i < entries.size(); i++) {
-            assertEquals(String.format("mmmmmmmmmmessage-%d", i), entries.get(i));
-        }
-    }
-
-    @Test
-    public void testCanReadEntryFromMLedgerWaitingForNewEntry() throws Exception {
-        AtomicInteger entryCount = new AtomicInteger(0);
-        AtomicBoolean entryProcessed = new AtomicBoolean(false);
-        Stack positions = new Stack<>();
-        List entries = new ArrayList<>();
-        for (int i = 0; i < 7; i++) {
-            ledger.addEntry(String.format("message-%d", i).getBytes(Encoding));
-        }
-
-        StreamingEntryReader streamingEntryReader =
-                new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic);
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                Entry entry = invocationOnMock.getArgument(0, Entry.class);
-                positions.push(entry.getPosition());
-                entries.add(new String(entry.getData()));
-                entryCount.getAndIncrement();
-                cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition()));
-                entryProcessed.set(true);
-                return null;
-            }
-        ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class));
-
-        streamingEntryReader.asyncReadEntries(5,  100, null);
-        await().until(() -> entryCount.get() == 5);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        streamingEntryReader.asyncReadEntries(5, 100, null);
-        // We only write 7 entries initially so only 7 entries can be read.
-        await().until(() -> entryCount.get() == 7);
-        // Add new entry and await for it to be send to reader.
-        entryProcessed.set(false);
-        ledger.addEntry("message-7".getBytes(Encoding));
-        await().until(() -> entryProcessed.get());
-        assertEquals(entries.size(), 8);
-        entryProcessed.set(false);
-        ledger.addEntry("message-8".getBytes(Encoding));
-        await().until(() -> entryProcessed.get());
-        assertEquals(entries.size(), 9);
-        entryProcessed.set(false);
-        ledger.addEntry("message-9".getBytes(Encoding));
-        await().until(() -> entryProcessed.get());
-        assertEquals(entries.size(), 10);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        for (int i = 0; i < entries.size(); i++) {
-            assertEquals(String.format("message-%d", i), entries.get(i));
-        }
-    }
-
-    @Test
-    public void testCanCancelReadEntryRequestAndResumeReading() throws Exception {
-        Map messages = new HashMap<>();
-        AtomicInteger count = new AtomicInteger(0);
-        Stack positions = new Stack<>();
-        List entries = new ArrayList<>();
-
-        for (int i = 0; i < 20; i++) {
-            String msg = String.format("message-%d", i);
-            messages.put(ledger.addEntry(msg.getBytes(Encoding)), msg);
-        }
-
-        StreamingEntryReader streamingEntryReader =
-                new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic);
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                Entry entry = invocationOnMock.getArgument(0, Entry.class);
-                positions.push(entry.getPosition());
-                entries.add(new String(entry.getData()));
-                cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition()));
-                return null;
-            }
-        ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class));
-
-        // Only return 5 entries
-        doAnswer(new Answer() {
-            @Override
-            public Void answer(InvocationOnMock invocationOnMock) {
-                AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class);
-                PositionImpl position = invocationOnMock.getArgument(0, PositionImpl.class);
-                int c = count.getAndIncrement();
-                if (c < 5) {
-                    cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(),
-                            messages.get(position).getBytes()),
-                            invocationOnMock.getArgument(2));
-                }
-                return null;
-            }
-        }).when(ledger).asyncReadEntry(any(), any(), any());
-
-        streamingEntryReader.asyncReadEntries(20,  200, null);
-        streamingEntryReader.cancelReadRequests();
-        await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Canceled);
-        // Only have 5 entry as we make ledger only return 5 entries and cancel the request.
-        assertEquals(entries.size(), 5);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        // Clear mock and try to read remaining entries
-        reset(ledger);
-        streamingEntryReader.asyncReadEntries(15,  200, null);
-        streamingEntryReader.cancelReadRequests();
-        await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Completed);
-        // Only have 5 entry as we make ledger only return 5 entries and cancel the request.
-        assertEquals(entries.size(), 20);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        // Make sure message still returned in order
-        for (int i = 0; i < entries.size(); i++) {
-            assertEquals(String.format("message-%d", i), entries.get(i));
-        }
-    }
-
-    @Test
-    public void testCanHandleExceptionAndRetry() throws Exception {
-        Map messages = new HashMap<>();
-        AtomicBoolean entryProcessed = new AtomicBoolean(false);
-        AtomicInteger count = new AtomicInteger(0);
-        Stack positions = new Stack<>();
-        List entries = new ArrayList<>();
-        for (int i = 0; i < 12; i++) {
-            String msg = String.format("message-%d", i);
-            messages.put(ledger.addEntry(msg.getBytes(Encoding)), msg);
-        }
-
-        StreamingEntryReader streamingEntryReader =
-                new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic);
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                Entry entry = invocationOnMock.getArgument(0, Entry.class);
-                positions.push(entry.getPosition());
-                entries.add(new String(entry.getData()));
-                cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition()));
-
-                if (entries.size() == 6 || entries.size() == 12) {
-                    entryProcessed.set(true);
-                }
-                return null;
-            }
-        ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class));
-
-        // Make reading from mledger throw exception randomly.
-        doAnswer(new Answer() {
-            @Override
-            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
-                AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class);
-                PositionImpl position = invocationOnMock.getArgument(0, PositionImpl.class);
-                int c = count.getAndIncrement();
-                if (c >= 3 && c < 5 || c >= 9 && c < 11) {
-                    cb.readEntryFailed(new ManagedLedgerException.TooManyRequestsException("Fake exception."),
-                            invocationOnMock.getArgument(2));
-                } else {
-                    cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(),
-                            messages.get(position).getBytes()),
-                            invocationOnMock.getArgument(2));
-                }
-                return null;
-            }
-        }).when(ledger).asyncReadEntry(any(), any(), any());
-
-        streamingEntryReader.asyncReadEntries(6,  100, null);
-        await().until(() -> entryProcessed.get());
-        assertEquals(entries.size(), 6);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        entryProcessed.set(false);
-        streamingEntryReader.asyncReadEntries(6, 100, null);
-        await().until(() -> entryProcessed.get());
-        assertEquals(entries.size(), 12);
-        assertEquals(cursor.getReadPosition(), ledger.getNextValidPosition((PositionImpl) positions.peek()));
-        // Make sure message still returned in order
-        for (int i = 0; i < entries.size(); i++) {
-            assertEquals(String.format("message-%d", i), entries.get(i));
-        }
-    }
-
-    @Test
-    public void testWillCancelReadAfterExhaustingRetry() throws Exception {
-        Map messages = new HashMap<>();
-        AtomicInteger count = new AtomicInteger(0);
-        Stack positions = new Stack<>();
-        List entries = new ArrayList<>();
-        for (int i = 0; i < 12; i++) {
-            String msg = String.format("message-%d", i);
-            messages.put(ledger.addEntry(msg.getBytes(Encoding)), msg);
-        }
-
-        StreamingEntryReader streamingEntryReader =
-                new StreamingEntryReader((ManagedCursorImpl) cursor, mockDispatcher, mockTopic);
-
-        doAnswer((InvocationOnMock invocationOnMock) -> {
-                    Entry entry = invocationOnMock.getArgument(0, Entry.class);
-                    positions.push(entry.getPosition());
-                    cursor.seek(ledger.getNextValidPosition((PositionImpl) entry.getPosition()));
-                    entries.add(new String(entry.getData()));
-                    return null;
-                }
-        ).when(mockDispatcher).readEntryComplete(any(Entry.class), any(PendingReadEntryRequest.class));
-
-        // Fail after first 3 read.
-        doAnswer(new Answer() {
-            @Override
-            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
-                AsyncCallbacks.ReadEntryCallback cb = invocationOnMock.getArgument(1, AsyncCallbacks.ReadEntryCallback.class);
-                PositionImpl position = invocationOnMock.getArgument(0, PositionImpl.class);
-                int c = count.getAndIncrement();
-                if (c >= 3) {
-                    cb.readEntryFailed(new ManagedLedgerException.TooManyRequestsException("Fake exception."),
-                            invocationOnMock.getArgument(2));
-                } else {
-                    cb.readEntryComplete(EntryImpl.create(position.getLedgerId(), position.getEntryId(),
-                            messages.get(position).getBytes()),
-                            invocationOnMock.getArgument(2));
-                }
-                return null;
-            }
-        }).when(ledger).asyncReadEntry(any(), any(), any());
-
-        streamingEntryReader.asyncReadEntries(5,  100, null);
-        await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Completed);
-        // Issued 5 read, should only have 3 entries as others were canceled after exhausting retries.
-        assertEquals(entries.size(), 3);
-        for (int i = 0; i < entries.size(); i++) {
-            assertEquals(String.format("message-%d", i), entries.get(i));
-        }
-        reset(ledger);
-        streamingEntryReader.asyncReadEntries(5,  100, null);
-        await().until(() -> streamingEntryReader.getState() == StreamingEntryReader.State.Completed);
-        assertEquals(entries.size(), 8);
-        for (int i = 0; i < entries.size(); i++) {
-            assertEquals(String.format("message-%d", i), entries.get(i));
-        }
-    }
-
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java
index ed85ffd600ec6..4f4affc39d316 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java
@@ -1228,7 +1228,6 @@ public void testBacklogConsumerCacheReads() throws Exception {
         conf.setManagedLedgerMinimumBacklogCursorsForCaching(2);
         conf.setManagedLedgerMinimumBacklogEntriesForCaching(10);
         conf.setManagedLedgerCacheEvictionTimeThresholdMillis(60 * 1000);
-        conf.setStreamingDispatch(false);
         restartBroker();
         final long totalMessages = 200;
         final int receiverSize = 10;

From b0bb5ae1e8606082a0511c71d36b7c6c53d7086d Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 17 May 2023 19:20:24 -0500
Subject: [PATCH 288/404] [fix][test] Use delta when comparing doubles in
 checkLoadReportNicSpeed (#20343)

### Motivation

@dave2wave recently encountered this test failure:

```
20:46:55  [ERROR] Tests run: 2, Failures: 1, Errors: 0, Skipped: 1, Time elapsed: 15.301 s <<< FAILURE! - in org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest
20:46:55  [ERROR] checkLoadReportNicSpeed(org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest)  Time elapsed: 0.142 s  <<< FAILURE!
20:46:55  java.lang.AssertionError: expected [1.6200000000000004E7] but found [1.62E7]
20:46:55  	at org.testng.Assert.fail(Assert.java:99)
20:46:55  	at org.testng.Assert.failNotEquals(Assert.java:1037)
20:46:55  	at org.testng.Assert.assertEquals(Assert.java:701)
20:46:55  	at org.testng.Assert.assertEquals(Assert.java:712)
20:46:55  	at org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest.checkLoadReportNicSpeed(LoadReportNetworkLimitTest.java:63)
```

The test failed because we are not providing an acceptable delta for the equality check.

It looks like there are other tests where we have strict double or float comparisons. I am going to leave those for now to minimize the diff on this PR. I read through all of the open flaky test issues and none appear to be related to this class of flakiness.

### Modifications

* Add a delta to the assertions in the `checkLoadReportNicSpeed` test

### Verifying this change

This change is a trivial rework / code cleanup without any test coverage.

### Documentation

- [x] `doc-not-needed`
---
 .../broker/loadbalance/LoadReportNetworkLimitTest.java    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java
index bec971dfa40e7..ffa2607b6b42e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java
@@ -60,12 +60,12 @@ public void checkLoadReportNicSpeed() throws Exception {
         LoadManagerReport report = admin.brokerStats().getLoadReport();
 
         if (SystemUtils.IS_OS_LINUX) {
-            assertEquals(report.getBandwidthIn().limit, usableNicCount * 5.4 * 1000 * 1000);
-            assertEquals(report.getBandwidthOut().limit, usableNicCount * 5.4 * 1000 * 1000);
+            assertEquals(report.getBandwidthIn().limit, usableNicCount * 5.4 * 1000 * 1000, 0.0001);
+            assertEquals(report.getBandwidthOut().limit, usableNicCount * 5.4 * 1000 * 1000, 0.0001);
         } else {
             // On non-Linux system we don't report the network usage
-            assertEquals(report.getBandwidthIn().limit, -1.0);
-            assertEquals(report.getBandwidthOut().limit, -1.0);
+            assertEquals(report.getBandwidthIn().limit, -1.0, 0.0001);
+            assertEquals(report.getBandwidthOut().limit, -1.0, 0.0001);
         }
     }
 

From e53fcaac3fde564e883fcc63baf74da60371ab5e Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Wed, 17 May 2023 20:33:16 -0500
Subject: [PATCH 289/404] [improve][sec] Suppress etcd CVE warnings (#20342)

---
 src/owasp-dependency-check-suppressions.xml | 68 +++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml
index a59b3b999e2bf..5a595af2c0a79 100644
--- a/src/owasp-dependency-check-suppressions.xml
+++ b/src/owasp-dependency-check-suppressions.xml
@@ -229,6 +229,74 @@
         CVE-2021-42550
     
 
+    
+    
+        
+        861af62ae22a71d30f401a80049397fe7ff44423
+        CVE-2020-15113
+    
+
+    
+        
+        663f2ccc0ec7797954c333fa75feeb7d559948b0
+        CVE-2020-15113
+    
+
+    
+        
+        abd0ffcd4e66046057c3bfb34affc0de870a038b
+        CVE-2020-15113
+    
+
+    
+        
+        a2e99802fec5586daca0c4daae975fe601a057a8
+        CVE-2017-8359
+    
+    
+        
+        a2e99802fec5586daca0c4daae975fe601a057a8
+        CVE-2020-15113
+    
+    
+        
+        a2e99802fec5586daca0c4daae975fe601a057a8
+        CVE-2020-7768
+    
+    
+        
+        a2e99802fec5586daca0c4daae975fe601a057a8
+        CVE-2017-7861
+    
+    
+        
+        a2e99802fec5586daca0c4daae975fe601a057a8
+        CVE-2017-9431
+    
+    
+        
+        a2e99802fec5586daca0c4daae975fe601a057a8
+        CVE-2017-7860
+    
+
     
     
         
Date: Thu, 18 May 2023 11:15:53 +0800
Subject: [PATCH 290/404] [fix][broker]Fix deadlock of metadata store (#20189)

Motivation: This task loadOrCreatePersistentTopic occupied the event thread of the ZK client so that other ZK tasks could not be finished anymore(Including the task itself), and it calls bundlesCache.synchronous().get(nsname) which is a blocking method.

Modification: Since the method getBundle(topic) will eventually call the method bundlesCache.synchronous().get(nsname), use getBundleAsync(topic) instead of getBundle(topic) to avoid blocking the thread.
---
 .../broker/namespace/NamespaceService.java    | 32 +++++++++++--------
 .../broker/namespace/OwnershipCache.java      |  6 ++--
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index a0e2bf7534c02..9d8d9e3890a19 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -38,7 +38,9 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Matcher;
@@ -1137,16 +1139,17 @@ public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName)
                 new IllegalArgumentException("Invalid class of NamespaceBundle: " + suName.getClass().getName()));
     }
 
+    /**
+     * @Deprecated This method is only used in test now.
+     */
+    @Deprecated
     public boolean isServiceUnitActive(TopicName topicName) {
         try {
-            OwnedBundle ownedBundle = ownershipCache.getOwnedBundle(getBundle(topicName));
-            if (ownedBundle == null) {
-                return false;
-            }
-            return ownedBundle.isActive();
-        } catch (Exception e) {
-            LOG.warn("Unable to find OwnedBundle for topic - [{}]", topicName, e);
-            return false;
+            return isServiceUnitActiveAsync(topicName).get(pulsar.getConfig()
+                    .getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+        } catch (InterruptedException | ExecutionException | TimeoutException e) {
+            LOG.warn("Unable to find OwnedBundle for topic in time - [{}]", topicName, e);
+            throw new RuntimeException(e);
         }
     }
 
@@ -1156,12 +1159,13 @@ public CompletableFuture isServiceUnitActiveAsync(TopicName topicName)
             return getBundleAsync(topicName)
                     .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.of(topicName), bundle));
         }
-        Optional> res = ownershipCache.getOwnedBundleAsync(getBundle(topicName));
-        if (!res.isPresent()) {
-            return CompletableFuture.completedFuture(false);
-        }
-
-        return res.get().thenApply(ob -> ob != null && ob.isActive());
+        return getBundleAsync(topicName).thenCompose(bundle -> {
+            Optional> optionalFuture = ownershipCache.getOwnedBundleAsync(bundle);
+            if (!optionalFuture.isPresent()) {
+                return CompletableFuture.completedFuture(false);
+            }
+            return optionalFuture.get().thenApply(ob -> ob != null && ob.isActive());
+        });
     }
 
     private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
index 7d0b5a4147721..86003153714cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
@@ -288,10 +288,10 @@ public Optional> getOwnedBundleAsync(NamespaceBun
 
     /**
      * Disable bundle in local cache and on zk.
-     *
-     * @param bundle
-     * @throws Exception
+     * @Deprecated This is a dangerous method  which is currently only used for test, it will occupy the ZK thread.
+     * Please switch to your own thread after calling this method.
      */
+    @Deprecated
     public CompletableFuture disableOwnership(NamespaceBundle bundle) {
         return updateBundleState(bundle, false)
                 .thenCompose(__ -> {

From 908d0b3f459167a3c3df2a85c23096df336427da Mon Sep 17 00:00:00 2001
From: Raghavender Mittapalli <14803749+syk-coder@users.noreply.github.com>
Date: Thu, 18 May 2023 09:10:25 +0530
Subject: [PATCH 291/404] [fix][broker] Use user-specified bundle size on
 creating a namespace anti-affinity group with the default local policies
 (#20327)

---
 .../apache/pulsar/broker/admin/impl/NamespacesBase.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 8c72d0b0286d3..97029eb5ce128 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -1710,7 +1710,8 @@ protected String internalGetNamespaceAntiAffinityGroup() {
         try {
             return getLocalPolicies()
                     .getLocalPolicies(namespaceName)
-                    .orElse(new LocalPolicies()).namespaceAntiAffinityGroup;
+                    .orElseGet(() -> new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles())
+                            , null, null)).namespaceAntiAffinityGroup;
         } catch (Exception e) {
             log.error("[{}] Failed to get the antiAffinityGroup of namespace {}", clientAppId(), namespaceName, e);
             throw new RestException(Status.NOT_FOUND, "Couldn't find namespace policies");
@@ -1760,7 +1761,9 @@ protected List internalGetAntiAffinityNamespaces(String cluster, String
                     throw new RuntimeException(e);
                 }
 
-                String storedAntiAffinityGroup = policies.orElse(new LocalPolicies()).namespaceAntiAffinityGroup;
+                String storedAntiAffinityGroup = policies.orElseGet(() ->
+                        new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()),
+                                null, null)).namespaceAntiAffinityGroup;
                 return antiAffinityGroup.equalsIgnoreCase(storedAntiAffinityGroup);
             }).collect(Collectors.toList());
 

From e4f1f095fef7e105cb7a3c5a1a8c2f9449888052 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 18 May 2023 00:27:54 -0500
Subject: [PATCH 292/404] [improve][ws] Use allowTopicOperationAsync for authz
 checks (#20299)

Similar to: https://github.com/apache/pulsar/pull/20142

### Motivation

In https://github.com/apache/pulsar/pull/20142 we changed the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method.

Our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, so this change helps us move in the right direction.

This PR follows the same logic and updates the WebSocket proxy to remove all calls to the `can*` methods in the `AuthorizationService`

### Modifications

* Update `ProducerHandler`,  `ConsumerHandler`, and `ReaderHander` in the WebSocket Proxy to call the `AuthorizationService#allowTopicOperationAsync` method.

### Verifying this change

This change is trivial.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: Skipping PR as I ran tests locally.
---
 .../pulsar/websocket/ConsumerHandler.java     | 20 +++++++++++++++++--
 .../pulsar/websocket/ProducerHandler.java     | 16 ++++++++++++++-
 .../pulsar/websocket/ReaderHandler.java       | 20 +++++++++++++++++--
 3 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
index c988fd1e70ce3..e326f2f02a901 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.websocket;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.base.Enums;
 import com.google.common.base.Splitter;
@@ -33,6 +34,7 @@
 import java.util.concurrent.atomic.LongAdder;
 import javax.servlet.http.HttpServletRequest;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.ConsumerBuilder;
 import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
@@ -47,6 +49,7 @@
 import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.client.impl.ConsumerBuilderImpl;
 import org.apache.pulsar.client.impl.TopicMessageIdImpl;
+import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.apache.pulsar.common.util.Codec;
 import org.apache.pulsar.common.util.DateFormatter;
 import org.apache.pulsar.websocket.data.ConsumerCommand;
@@ -467,8 +470,21 @@ protected ConsumerBuilder getConsumerConfiguration(PulsarClient client)
 
     @Override
     protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception {
-        return service.getAuthorizationService().canConsume(topic, authRole, authenticationData,
-                this.subscription);
+        try {
+            AuthenticationDataSubscription subscription = new AuthenticationDataSubscription(authenticationData,
+                    this.subscription);
+            return service.getAuthorizationService()
+                    .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription)
+                    .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+        } catch (InterruptedException e) {
+            log.warn("Time-out {} sec while checking authorization on {} ",
+                    service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic);
+            throw e;
+        } catch (Exception e) {
+            log.warn("Consumer-client  with Role - {} failed to get permissions for topic - {}. {}", authRole, topic,
+                    e.getMessage());
+            throw e;
+        }
     }
 
     public static String extractSubscription(HttpServletRequest request) {
diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java
index 1dc3f202fe07d..f18d46e285617 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java
@@ -20,6 +20,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.pulsar.websocket.WebSocketError.FailedToDeserializeFromJSON;
 import static org.apache.pulsar.websocket.WebSocketError.PayloadEncodingError;
 import static org.apache.pulsar.websocket.WebSocketError.UnknownError;
@@ -45,6 +46,7 @@
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.SchemaSerializationException;
 import org.apache.pulsar.client.api.TypedMessageBuilder;
+import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.apache.pulsar.common.util.DateFormatter;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.websocket.data.ProducerAck;
@@ -242,7 +244,19 @@ public long getMsgPublishedCounter() {
 
     @Override
     protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception {
-        return service.getAuthorizationService().canProduce(topic, authRole, authenticationData);
+        try {
+            return service.getAuthorizationService()
+                    .allowTopicOperationAsync(topic, TopicOperation.PRODUCE, authRole, authenticationData)
+                    .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+        } catch (InterruptedException e) {
+            log.warn("Time-out {} sec while checking authorization on {} ",
+                    service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic);
+            throw e;
+        } catch (Exception e) {
+            log.warn("Producer-client  with Role - {} failed to get permissions for topic - {}. {}", authRole, topic,
+                    e.getMessage());
+            throw e;
+        }
     }
 
     private void sendAckResponse(ProducerAck response) {
diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java
index 890f426a9bbf3..3d726b9f02f76 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.websocket;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.IOException;
@@ -28,6 +29,7 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
 import org.apache.pulsar.client.api.MessageId;
@@ -38,6 +40,7 @@
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.client.impl.MultiTopicsReaderImpl;
 import org.apache.pulsar.client.impl.ReaderImpl;
+import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.apache.pulsar.common.util.DateFormatter;
 import org.apache.pulsar.websocket.data.ConsumerCommand;
 import org.apache.pulsar.websocket.data.ConsumerMessage;
@@ -310,8 +313,21 @@ protected void updateDeliverMsgStat(long msgSize) {
 
     @Override
     protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception {
-        return service.getAuthorizationService().canConsume(topic, authRole, authenticationData,
-                this.subscription);
+        try {
+            AuthenticationDataSubscription subscription = new AuthenticationDataSubscription(authenticationData,
+                    this.subscription);
+            return service.getAuthorizationService()
+                    .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription)
+                    .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+        } catch (InterruptedException e) {
+            log.warn("Time-out {} sec while checking authorization on {} ",
+                    service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic);
+            throw e;
+        } catch (Exception e) {
+            log.warn("Consumer-client  with Role - {} failed to get permissions for topic - {}. {}", authRole, topic,
+                    e.getMessage());
+            throw e;
+        }
     }
 
     private int getReceiverQueueSize() {

From 65f61121e99b504294cf375f319f7946ff4c50fd Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 18 May 2023 09:46:52 -0500
Subject: [PATCH 293/404] [cleanup] Catch TimeoutException when logging about
 time outs (#20349)

Relates to: #20299

### Motivation

We have several catch blocks that are dedicated to providing meaningful logs about timeouts. As such, we should catch the `TimeoutException`, not the `InterruptedException`, in order to ensure accurate logs.

### Modifications

* Replace `InterruptedException` with `TimeoutException`

### Verifying this change

This change is a trivial rework / code cleanup without any test coverage.

### Documentation

- [x] `doc-not-needed`
---
 .../pulsar/broker/authorization/AuthorizationService.java  | 7 ++++---
 .../java/org/apache/pulsar/broker/rest/TopicsBase.java     | 3 ++-
 .../java/org/apache/pulsar/websocket/ConsumerHandler.java  | 3 ++-
 .../java/org/apache/pulsar/websocket/ProducerHandler.java  | 3 ++-
 .../java/org/apache/pulsar/websocket/ReaderHandler.java    | 3 ++-
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index 0c61219b57a50..6f303e2117fe0 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -23,6 +23,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
 import javax.ws.rs.core.Response;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
@@ -221,7 +222,7 @@ public boolean canProduce(TopicName topicName, String role, AuthenticationDataSo
         try {
             return canProduceAsync(topicName, role, authenticationData).get(
                     conf.getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-        } catch (InterruptedException e) {
+        } catch (TimeoutException e) {
             log.warn("Time-out {} sec while checking authorization on {} ",
                     conf.getMetadataStoreOperationTimeoutSeconds(), topicName);
             throw e;
@@ -237,7 +238,7 @@ public boolean canConsume(TopicName topicName, String role, AuthenticationDataSo
         try {
             return canConsumeAsync(topicName, role, authenticationData, subscription)
                     .get(conf.getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-        } catch (InterruptedException e) {
+        } catch (TimeoutException e) {
             log.warn("Time-out {} sec while checking authorization on {} ",
                     conf.getMetadataStoreOperationTimeoutSeconds(), topicName);
             throw e;
@@ -263,7 +264,7 @@ public boolean canLookup(TopicName topicName, String role, AuthenticationDataSou
         try {
             return canLookupAsync(topicName, role, authenticationData)
                     .get(conf.getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-        } catch (InterruptedException e) {
+        } catch (TimeoutException e) {
             log.warn("Time-out {} sec while checking authorization on {} ",
                     conf.getMetadataStoreOperationTimeoutSeconds(), topicName);
             throw e;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
index 7d3aa37fa4ec5..6f3ac7f8c09ca 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
@@ -39,6 +39,7 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 import javax.ws.rs.container.AsyncResponse;
 import javax.ws.rs.core.Response;
@@ -771,7 +772,7 @@ && pulsar().getBrokerService().isAuthorizationEnabled()) {
                 isAuthorized = pulsar().getBrokerService().getAuthorizationService()
                         .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, authParams)
                         .get(config().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-            } catch (InterruptedException e) {
+            } catch (TimeoutException e) {
                 log.warn("Time-out {} sec while checking authorization on {} ",
                         config().getMetadataStoreOperationTimeoutSeconds(), topicName);
                 throw new RestException(Status.INTERNAL_SERVER_ERROR, "Time-out while checking authorization");
diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
index e326f2f02a901..2ab62b10ee9ee 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java
@@ -29,6 +29,7 @@
 import java.util.Base64;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.atomic.LongAdder;
@@ -476,7 +477,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent
             return service.getAuthorizationService()
                     .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription)
                     .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-        } catch (InterruptedException e) {
+        } catch (TimeoutException e) {
             log.warn("Time-out {} sec while checking authorization on {} ",
                     service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic);
             throw e;
diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java
index f18d46e285617..5ad1283fe84c4 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java
@@ -34,6 +34,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.atomic.LongAdder;
 import javax.servlet.http.HttpServletRequest;
@@ -248,7 +249,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent
             return service.getAuthorizationService()
                     .allowTopicOperationAsync(topic, TopicOperation.PRODUCE, authRole, authenticationData)
                     .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-        } catch (InterruptedException e) {
+        } catch (TimeoutException e) {
             log.warn("Time-out {} sec while checking authorization on {} ",
                     service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic);
             throw e;
diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java
index 3d726b9f02f76..2f985b2076da2 100644
--- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java
+++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java
@@ -23,6 +23,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.IOException;
 import java.util.Base64;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.atomic.LongAdder;
@@ -319,7 +320,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent
             return service.getAuthorizationService()
                     .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription)
                     .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-        } catch (InterruptedException e) {
+        } catch (TimeoutException e) {
             log.warn("Time-out {} sec while checking authorization on {} ",
                     service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic);
             throw e;

From c0757254a0ea2625ffc6d97687a715201e20ef2f Mon Sep 17 00:00:00 2001
From: tison 
Date: Thu, 18 May 2023 23:19:10 +0800
Subject: [PATCH 294/404] [improve][admin] Return BAD_REQUEST on cluster data
 is null for createCluster (#20346)

---
 .../java/org/apache/pulsar/broker/admin/impl/ClustersBase.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java
index e61a7f20d1d67..5d4ed54c33466 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java
@@ -162,6 +162,9 @@ public void createCluster(
                 .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync())
                 .thenCompose(__ -> {
                     NamedEntity.checkName(cluster);
+                    if (clusterData == null) {
+                        throw new RestException(Status.BAD_REQUEST, "cluster data is required");
+                    }
                     try {
                         clusterData.checkPropertiesIfPresent();
                     } catch (IllegalArgumentException ex) {

From d0d626be8f59107d9de624f813dd0d5012cc1311 Mon Sep 17 00:00:00 2001
From: "Kim, Joo Hyuk" 
Date: Fri, 19 May 2023 00:21:37 +0900
Subject: [PATCH 295/404] [improve][cli] Allow pulser-client consume create a
 replicated subscription (#20316)

---
 .../main/java/org/apache/pulsar/client/cli/CmdConsume.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java
index 58ab6360a17bb..0c65604cbe6b8 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java
@@ -109,6 +109,9 @@ public class CmdConsume extends AbstractCmdConsume {
     @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1)
     private boolean poolMessages = true;
 
+    @Parameter(names = {"-rs", "--replicated" }, description = "Whether the subscription status should be replicated")
+    private boolean replicateSubscriptionState = false;
+
     public CmdConsume() {
         // Do nothing
         super();
@@ -156,7 +159,8 @@ private int consume(String topic) {
                     .subscriptionType(subscriptionType)
                     .subscriptionMode(subscriptionMode)
                     .subscriptionInitialPosition(subscriptionInitialPosition)
-                    .poolMessages(poolMessages);
+                    .poolMessages(poolMessages)
+                    .replicateSubscriptionState(replicateSubscriptionState);
 
             if (isRegex) {
                 builder.topicsPattern(Pattern.compile(topic));

From 43a989862f548fa3f67708a5fff62eb764af878c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gabriel=20Mikl=C3=B3s?= 
Date: Thu, 18 May 2023 17:23:55 +0200
Subject: [PATCH 296/404] [improve][fn] Use functions classloader in
 TopicSchema.newSchemaInstance() to fix ClassNotFoundException when using
 custom SerDe classes. (targeted for master) (#20115)

---
 .../pulsar/functions/instance/ContextImpl.java    |  2 +-
 .../apache/pulsar/functions/sink/PulsarSink.java  |  2 +-
 .../pulsar/functions/source/PulsarSource.java     |  6 +++---
 .../source/SingleConsumerPulsarSource.java        | 11 ++++-------
 .../pulsar/functions/source/TopicSchema.java      | 15 ++++++++++++---
 .../pulsar/functions/source/TopicSchemaTest.java  |  8 +++++---
 .../functions/PulsarFunctionsTest.java            |  8 ++++----
 .../functions/PulsarFunctionsTestBase.java        |  2 +-
 .../functions/java/PulsarFunctionsJavaTest.java   | 11 +++++++----
 .../java/PulsarWorkerRebalanceDrainTest.java      | 15 +++++++++------
 .../functions/utils/CommandGenerator.java         | 10 +++++-----
 11 files changed, 52 insertions(+), 38 deletions(-)

diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
index d64c5f9b52db6..5cbbcad24c7a2 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
@@ -148,7 +148,7 @@ public ContextImpl(InstanceConfig config, Logger logger, PulsarClient client,
         this.clientBuilder = clientBuilder;
         this.client = client;
         this.pulsarAdmin = pulsarAdmin;
-        this.topicSchema = new TopicSchema(client);
+        this.topicSchema = new TopicSchema(client, Thread.currentThread().getContextClassLoader());
         this.statsManager = statsManager;
 
         this.producerBuilder = (ProducerBuilderImpl) client.newProducer().blockIfQueueFull(true).enableBatching(true)
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
index 8add0a78c5fff..97a0ad0a2ce17 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
@@ -349,7 +349,7 @@ public PulsarSink(PulsarClient client, PulsarSinkConfig pulsarSinkConfig, Map buildPulsarSourceConsumerConfig(String t
                                                                             Class typeArg) {
         PulsarSourceConsumerConfig.PulsarSourceConsumerConfigBuilder consumerConfBuilder =
                 PulsarSourceConsumerConfig.builder().isRegexPattern(conf.isRegexPattern())
-                .receiverQueueSize(conf.getReceiverQueueSize())
-                .consumerProperties(conf.getConsumerProperties());
+                        .receiverQueueSize(conf.getReceiverQueueSize())
+                        .consumerProperties(conf.getConsumerProperties());
 
         Schema schema;
         if (conf.getSerdeClassName() != null && !conf.getSerdeClassName().isEmpty()) {
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java
index 426723804cad1..d4d3ea00b9317 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java
@@ -44,14 +44,12 @@ public class SingleConsumerPulsarSource extends PulsarSource {
     private Consumer consumer;
     private final List> inputConsumers = new LinkedList<>();
 
-    public SingleConsumerPulsarSource(PulsarClient pulsarClient,
-                                      SingleConsumerPulsarSourceConfig pulsarSourceConfig,
-                                      Map properties,
-                                      ClassLoader functionClassLoader) {
+    public SingleConsumerPulsarSource(PulsarClient pulsarClient, SingleConsumerPulsarSourceConfig pulsarSourceConfig,
+                                      Map properties, ClassLoader functionClassLoader) {
         super(pulsarClient, pulsarSourceConfig, properties, functionClassLoader);
         this.pulsarClient = pulsarClient;
         this.pulsarSourceConfig = pulsarSourceConfig;
-        this.topicSchema = new TopicSchema(pulsarClient);
+        this.topicSchema = new TopicSchema(pulsarClient, functionClassLoader);
         this.properties = properties;
         this.functionClassLoader = functionClassLoader;
     }
@@ -60,8 +58,7 @@ public SingleConsumerPulsarSource(PulsarClient pulsarClient,
     public void open(Map config, SourceContext sourceContext) throws Exception {
         log.info("Opening pulsar source with config: {}", pulsarSourceConfig);
 
-        Class typeArg = Reflections.loadClass(this.pulsarSourceConfig.getTypeClassName(),
-                this.functionClassLoader);
+        Class typeArg = Reflections.loadClass(this.pulsarSourceConfig.getTypeClassName(), this.functionClassLoader);
 
         checkArgument(!Void.class.equals(typeArg), "Input type of Pulsar Function cannot be Void");
 
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
index 8ed177cba3b19..d8ae6b19f4a7e 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
@@ -19,7 +19,11 @@
 package org.apache.pulsar.functions.source;
 
 import io.netty.buffer.ByteBuf;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.nio.ByteBuffer;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -47,8 +51,13 @@ public class TopicSchema {
     private final Map> cachedSchemas = new HashMap<>();
     private final PulsarClient client;
 
-    public TopicSchema(PulsarClient client) {
+    private final ClassLoader functionsClassloader;
+
+    public TopicSchema(PulsarClient client, ClassLoader functionsClassloader) {
         this.client = client;
+        this.functionsClassloader = AccessController.doPrivileged(
+                (PrivilegedAction) () -> new URLClassLoader(new URL[0], functionsClassloader)
+        );
     }
 
     /**
@@ -242,11 +251,11 @@ private  Schema newSchemaInstance(String topic, Class clazz, ConsumerCo
     @SuppressWarnings("unchecked")
     private  Schema newSchemaInstance(String topic, Class clazz, String schemaTypeOrClassName, boolean input) {
         return newSchemaInstance(topic, clazz, new ConsumerConfig(schemaTypeOrClassName), input,
-                Thread.currentThread().getContextClassLoader());
+                functionsClassloader);
     }
 
     @SuppressWarnings("unchecked")
     private  Schema newSchemaInstance(String topic, Class clazz, ConsumerConfig conf, boolean input) {
-        return newSchemaInstance(topic, clazz, conf, input, Thread.currentThread().getContextClassLoader());
+        return newSchemaInstance(topic, clazz, conf, input, functionsClassloader);
     }
 }
diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java
index 506ef67ab0740..c4e77cb3ff85d 100644
--- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java
+++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java
@@ -18,8 +18,6 @@
  */
 package org.apache.pulsar.functions.source;
 
-import static org.testng.Assert.assertEquals;
-import java.util.Optional;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.impl.schema.AvroSchema;
@@ -30,12 +28,16 @@
 import org.apache.pulsar.functions.proto.Request;
 import org.testng.annotations.Test;
 
+import java.util.Optional;
+
+import static org.testng.Assert.assertEquals;
+
 @Slf4j
 public class TopicSchemaTest {
 
     @Test
     public void testGetSchema() {
-        TopicSchema topicSchema = new TopicSchema(null);
+        TopicSchema topicSchema = new TopicSchema(null, Thread.currentThread().getContextClassLoader());
 
         String TOPIC = "public/default/test";
         Schema schema = topicSchema.getSchema(TOPIC + "1", DummyClass.class, Optional.of(SchemaType.JSON));
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
index 6088628aac52b..1e54764ad5d2b 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
@@ -24,10 +24,10 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.swagger.util.Json;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,8 +43,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-
-import io.swagger.util.Json;
 import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -77,8 +75,8 @@
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.functions.api.examples.AutoSchemaFunction;
 import org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction;
-import org.apache.pulsar.functions.api.examples.MergeTopicFunction;
 import org.apache.pulsar.functions.api.examples.InitializableFunction;
+import org.apache.pulsar.functions.api.examples.MergeTopicFunction;
 import org.apache.pulsar.functions.api.examples.RecordFunction;
 import org.apache.pulsar.functions.api.examples.pojo.AvroTestObject;
 import org.apache.pulsar.functions.api.examples.pojo.Users;
@@ -902,6 +900,7 @@ protected  void submitFunction(Runtime runtime,
                                       String functionName,
                                       String functionFile,
                                       String functionClass,
+                                      Map inputSerdeClassNames,
                                       String outputSerdeClassName,
                                       Map userConfigs) throws Exception {
 
@@ -916,6 +915,7 @@ protected  void submitFunction(Runtime runtime,
         }
         generator.setSinkTopic(outputTopicName);
         generator.setFunctionName(functionName);
+        generator.setCustomSerDeSourceTopics(inputSerdeClassNames);
         generator.setOutputSerDe(outputSerdeClassName);
         if (userConfigs != null) {
             generator.setUserConfig(userConfigs);
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
index 62aa36da1b6be..288ced63ae5eb 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
@@ -54,7 +54,7 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite {
     public static final String SERDE_JAVA_CLASS =
             "org.apache.pulsar.functions.api.examples.CustomBaseToBaseFunction";
 
-    public static final String SERDE_OUTPUT_CLASS =
+    public static final String SERDE_CLASS =
             "org.apache.pulsar.functions.api.examples.CustomBaseSerde";
 
     public static final String EXCLAMATION_PYTHON_CLASS =
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java
index 097a452937d27..939d6e19d1fb1 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java
@@ -19,9 +19,9 @@
 package org.apache.pulsar.tests.integration.functions.java;
 
 import static org.testng.Assert.assertEquals;
-
 import java.util.Collections;
-
+import java.util.Map;
+import org.apache.commons.collections4.map.HashedMap;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.common.policies.data.FunctionStatus;
 import org.apache.pulsar.common.policies.data.FunctionStatusUtil;
@@ -70,10 +70,13 @@ private void testCustomSerdeFunction() throws Exception {
             admin.topics().createNonPartitionedTopic(outputTopicName);
         }
 
+        Map inputTopicsSerde = new HashedMap<>();
+        inputTopicsSerde.put(inputTopicName, SERDE_CLASS);
+
         String functionName = "test-serde-fn-" + randomName(8);
         submitFunction(
-                Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS,
-                SERDE_OUTPUT_CLASS, Collections.singletonMap("serde-topic", outputTopicName)
+                Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, inputTopicsSerde,
+                SERDE_CLASS, Collections.singletonMap("serde-topic", outputTopicName)
         );
 
         // get function info
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java
index a6f54349f2689..de29cbc94c295 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java
@@ -18,6 +18,9 @@
  */
 package org.apache.pulsar.tests.integration.functions.java;
 
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import com.fasterxml.jackson.databind.MappingIterator;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -26,10 +29,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import com.fasterxml.jackson.databind.MappingIterator;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
+import org.apache.commons.collections4.map.HashedMap;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.common.functions.WorkerInfo;
 import org.apache.pulsar.common.policies.data.FunctionStatus;
@@ -41,8 +43,6 @@
 import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType;
 import org.apache.pulsar.tests.integration.topologies.PulsarCluster;
 import org.testng.annotations.Test;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
 
 @Slf4j
 public abstract class PulsarWorkerRebalanceDrainTest extends PulsarFunctionsTest {
@@ -251,9 +251,12 @@ private void createFunctionWorker(String functionName, String topicPrefix) throw
             admin.topics().createNonPartitionedTopic(outputTopicName);
         }
 
+        Map inputTopicsSerde = new HashedMap<>();
+        inputTopicsSerde.put(inputTopicName, SERDE_CLASS);
+
         submitFunction(
-                Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS,
-                SERDE_OUTPUT_CLASS, Collections.singletonMap(topicPrefix, outputTopicName)
+                Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, inputTopicsSerde,
+                SERDE_CLASS, Collections.singletonMap(topicPrefix, outputTopicName)
         );
     }
 
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
index 90fac7a055268..adc791fab4dc8 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
@@ -46,7 +46,7 @@ public enum Runtime {
     private String functionClassName;
     private String sourceTopic;
     private String sourceTopicPattern;
-    private Map customSereSourceTopics;
+    private Map customSerDeSourceTopics;
     private String sinkTopic;
     private String logTopic;
     private String outputSerDe;
@@ -182,8 +182,8 @@ public String generateCreateFunctionCommand(String codeFile) {
         if (batchBuilder != null) {
             commandBuilder.append("--batch-builder" + batchBuilder);
         }
-        if (customSereSourceTopics != null && !customSereSourceTopics.isEmpty()) {
-            commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSereSourceTopics) + "\'");
+        if (customSerDeSourceTopics != null && !customSerDeSourceTopics.isEmpty()) {
+            commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSerDeSourceTopics) + "\'");
         }
         if (sinkTopic != null) {
             commandBuilder.append(" --output " + sinkTopic);
@@ -280,8 +280,8 @@ public String generateUpdateFunctionCommand(String codeFile) {
         if (StringUtils.isNotEmpty(sourceTopic)) {
             commandBuilder.append(" --inputs " + sourceTopic);
         }
-        if (customSereSourceTopics != null && !customSereSourceTopics.isEmpty()) {
-            commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSereSourceTopics) + "\'");
+        if (customSerDeSourceTopics != null && !customSerDeSourceTopics.isEmpty()) {
+            commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSerDeSourceTopics) + "\'");
         }
         if (batchBuilder != null) {
             commandBuilder.append("--batch-builder" + batchBuilder);

From 2ebb3797c3f371c3ca22cbc8002a8110e3e3fa47 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 18 May 2023 13:13:13 -0500
Subject: [PATCH 297/404] [fix][test] ProxyWithoutServiceDiscoveryTest should
 enable authz (#20348)

---
 .../ProxyWithoutServiceDiscoveryTest.java     | 52 +++++++++----------
 1 file changed, 25 insertions(+), 27 deletions(-)

diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java
index 6a61decfcc693..9c8e2ba33c9e8 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java
@@ -54,13 +54,6 @@
 
 public class ProxyWithoutServiceDiscoveryTest extends ProducerConsumerBase {
     private static final Logger log = LoggerFactory.getLogger(ProxyWithoutServiceDiscoveryTest.class);
-
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
-    private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-
     private ProxyService proxyService;
     private ProxyConfiguration proxyConfig = new ProxyConfiguration();
 
@@ -70,22 +63,27 @@ protected void setup() throws Exception {
 
         // enable tls and auth&auth at broker
         conf.setAuthenticationEnabled(true);
-        conf.setAuthorizationEnabled(false);
+        conf.setAuthorizationEnabled(true);
 
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        conf.setTlsAllowInsecureConnection(true);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
 
         Set superUserRoles = new HashSet<>();
-        superUserRoles.add("superUser");
+        superUserRoles.add("admin");
+        superUserRoles.add("superproxy");
         conf.setSuperUserRoles(superUserRoles);
 
+        Set proxyRoles = new HashSet<>();
+        proxyRoles.add("superproxy");
+        conf.setProxyRoles(proxyRoles);
+
+        conf.setBrokerClientTlsEnabled(true);
         conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
-        conf.setBrokerClientAuthenticationParameters(
-                "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SERVER_KEY_FILE_PATH);
+        conf.setBrokerClientAuthenticationParameters(String.format("tlsCertFile:%s,tlsKeyFile:%s",
+                getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8")));
 
         Set providers = new HashSet<>();
         providers.add(AuthenticationProviderTls.class.getName());
@@ -110,14 +108,14 @@ protected void setup() throws Exception {
         proxyConfig.setTlsEnabledWithBroker(true);
 
         // enable tls and auth&auth at proxy
-        proxyConfig.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        proxyConfig.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-        proxyConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
+        proxyConfig.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
 
         proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
-        proxyConfig.setBrokerClientAuthenticationParameters(
-                "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH);
-        proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        proxyConfig.setBrokerClientAuthenticationParameters(String.format("tlsCertFile:%s,tlsKeyFile:%s",
+                getTlsFileForClient("superproxy.cert"), getTlsFileForClient("superproxy.key-pk8")));
+        proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
 
         proxyConfig.setAuthenticationProviders(providers);
 
@@ -137,7 +135,7 @@ protected void cleanup() throws Exception {
 
     /**
      * 
    -     * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
+     * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker)
      *
      * 1. client connects to proxy over tls and pass auth-data
      * 2. proxy authenticate client and retrieve client-role
@@ -154,8 +152,8 @@ public void testDiscoveryService() throws Exception {
         log.info("-- Starting {} test --", methodName);
 
         Map authParams = Maps.newHashMap();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
         Authentication authTls = new AuthenticationTls();
         authTls.configure(authParams);
         // create a client which connects to proxy over tls and pass authData
@@ -198,10 +196,10 @@ public void testDiscoveryService() throws Exception {
     }
 
     protected final PulsarClient createPulsarClient(Authentication auth, String lookupUrl) throws Exception {
-        admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
-                .allowTlsInsecureConnection(true).authentication(auth).build());
+        admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()).tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
+                .authentication(auth).build());
         return PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
-                .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth)
+                .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth)
                 .enableTls(true).build();
     }
 

From d565c953a7deab1ea93949bfefa44477e51e417f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Fri, 19 May 2023 11:19:14 -0500
Subject: [PATCH 298/404] [cleanup] Consolidate certs in broker (and some
 proxy) tests (#20353)

---
 .../pulsar/broker/admin/AdminApiTest.java     |   9 +-
 .../broker/admin/v1/V1_AdminApiTest.java      |   9 +-
 .../broker/service/BrokerServiceTest.java     |  41 +++---
 .../pulsar/broker/web/WebServiceTest.java     |  30 +++--
 .../client/api/BrokerServiceLookupTest.java   |  44 ++----
 .../worker/PulsarFunctionTlsTest.java         |  16 ++-
 .../authentication/tls/broker-cert.pem        | 117 ----------------
 .../authentication/tls/broker-key.pem         |  27 ----
 .../resources/authentication/tls/cacert.pem   | 127 ------------------
 .../authentication/tls/client-cert.pem        |  90 -------------
 .../authentication/tls/client-key.pem         |  27 ----
 .../src/test/resources/certificate/client.crt |  20 ---
 .../src/test/resources/certificate/client.csr |  17 ---
 .../src/test/resources/certificate/client.key |  28 ----
 .../src/test/resources/certificate/server.crt |  20 ---
 .../src/test/resources/certificate/server.csr |  17 ---
 .../src/test/resources/certificate/server.key |  28 ----
 .../client/cli/PulsarClientToolTest.java      |   9 +-
 .../server/ProxyServiceTlsStarterTest.java    |  16 +--
 .../pulsar/proxy/server/ProxyTlsTest.java     |  12 +-
 .../proxy/server/ProxyTlsTestWithAuth.java    |   8 +-
 21 files changed, 84 insertions(+), 628 deletions(-)
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/broker-key.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/cacert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/client-cert.pem
 delete mode 100644 pulsar-broker/src/test/resources/authentication/tls/client-key.pem
 delete mode 100644 pulsar-broker/src/test/resources/certificate/client.crt
 delete mode 100644 pulsar-broker/src/test/resources/certificate/client.csr
 delete mode 100644 pulsar-broker/src/test/resources/certificate/client.key
 delete mode 100644 pulsar-broker/src/test/resources/certificate/server.crt
 delete mode 100644 pulsar-broker/src/test/resources/certificate/server.csr
 delete mode 100644 pulsar-broker/src/test/resources/certificate/server.key

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index f0ac63f08ae4a..855343e18a24b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -154,9 +154,6 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {
 
     private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class);
 
-    private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
-    private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-
     private MockedPulsarService mockPulsarSetup;
 
     private PulsarService otherPulsar;
@@ -185,8 +182,8 @@ private void applyDefaultConfig() {
         conf.setLoadBalancerEnabled(true);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setMessageExpiryCheckIntervalInMinutes(1);
         conf.setSubscriptionExpiryCheckIntervalInMinutes(1);
         conf.setBrokerDeleteInactiveTopicsEnabled(false);
@@ -203,7 +200,7 @@ private void setupConfigAndStart(java.util.function.Consumer authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         PulsarClient pulsarClient = null;
 
@@ -852,15 +847,15 @@ public void testTlsAuthDisallowInsecure() throws Exception {
         conf.setAuthenticationProviders(providers);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(false);
         conf.setNumExecutorThreadPoolSize(5);
         restartBroker();
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         PulsarClient pulsarClient = null;
 
@@ -914,16 +909,16 @@ public void testTlsAuthUseTrustCert() throws Exception {
         conf.setAuthenticationProviders(providers);
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setTlsAllowInsecureConnection(false);
-        conf.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
         conf.setNumExecutorThreadPoolSize(5);
         restartBroker();
 
         Map authParams = new HashMap<>();
-        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+        authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
+        authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));
 
         PulsarClient pulsarClient = null;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
index ca8efe9d1cc79..b069d31dc6e0d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
@@ -63,6 +63,7 @@
 import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.common.util.SecurityUtility;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.asynchttpclient.AsyncHttpClient;
 import org.asynchttpclient.BoundRequestBuilder;
 import org.asynchttpclient.DefaultAsyncHttpClient;
@@ -84,10 +85,17 @@ public class WebServiceTest {
     private PulsarService pulsar;
     private String BROKER_LOOKUP_URL;
     private String BROKER_LOOKUP_URL_TLS;
-    private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
-    private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-    private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
-    private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
+
+    private final static String CA_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
+    private final static String BROKER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final static String BROKER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final static String CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final static String CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
 
 
     @Test
@@ -351,8 +359,8 @@ private String makeHttpRequest(boolean useTls, boolean useAuth) throws Exception
             if (useTls) {
                 KeyManager[] keyManagers = null;
                 if (useAuth) {
-                    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
-                    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
+                    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(CLIENT_CERT_FILE_PATH);
+                    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(CLIENT_KEY_FILE_PATH);
 
                     KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
                     ks.load(null, null);
@@ -403,10 +411,10 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut
         config.setAuthenticationProviders(providers);
         config.setAuthorizationEnabled(false);
         config.setSuperUserRoles(roles);
-        config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         config.setTlsAllowInsecureConnection(allowInsecure);
-        config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
+        config.setTlsTrustCertsFilePath(allowInsecure ? "" : CA_CERT_FILE_PATH);
         config.setClusterName("local");
         config.setAdvertisedAddress("localhost"); // TLS certificate expects localhost
         config.setMetadataStoreUrl("zk:localhost:2181");
@@ -433,8 +441,8 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut
             serviceUrl = BROKER_URL_BASE_TLS;
 
             Map authParams = new HashMap<>();
-            authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
-            authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
+            authParams.put("tlsCertFile", CLIENT_CERT_FILE_PATH);
+            authParams.put("tlsKeyFile", CLIENT_KEY_FILE_PATH);
 
             adminBuilder.authentication(AuthenticationTls.class.getName(), authParams).allowTlsInsecureConnection(true);
         }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
index 8597e0a87997a..792f419ee997e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
@@ -31,7 +31,6 @@
 import com.google.common.util.concurrent.MoreExecutors;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Field;
@@ -41,10 +40,6 @@
 import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
-import java.security.KeyStore;
-import java.security.PrivateKey;
-import java.security.SecureRandom;
-import java.security.cert.Certificate;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -62,10 +57,7 @@
 import java.util.stream.Collectors;
 import javax.naming.AuthenticationException;
 import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
 import lombok.Cleanup;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.PulsarService;
@@ -427,10 +419,6 @@ public void testPartitionTopicLookup() throws Exception {
     @Test
     public void testWebserviceServiceTls() throws Exception {
         log.info("-- Starting {} test --", methodName);
-        final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
-        final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-        final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
-        final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
 
         /**** start broker-2 ****/
         ServiceConfiguration conf2 = new ServiceConfiguration();
@@ -443,12 +431,15 @@ public void testWebserviceServiceTls() throws Exception {
         conf2.setWebServicePort(Optional.of(0));
         conf2.setWebServicePortTls(Optional.of(0));
         conf2.setAdvertisedAddress("localhost");
-        conf2.setTlsAllowInsecureConnection(true);
-        conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf2.setTlsRequireTrustedClientCertOnConnect(true);
+        conf2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf2.setClusterName(conf.getClusterName());
         conf2.setMetadataStoreUrl("zk:localhost:2181");
         conf2.setConfigurationMetadataStoreUrl("zk:localhost:3181");
+        // Not in use, and because TLS is not configured, it will fail to start
+        conf2.setSystemTopicEnabled(false);
 
         @Cleanup
         PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf2);
@@ -457,10 +448,13 @@ public void testWebserviceServiceTls() throws Exception {
         // restart broker1 with tls enabled
         conf.setBrokerServicePortTls(Optional.of(0));
         conf.setWebServicePortTls(Optional.of(0));
-        conf.setTlsAllowInsecureConnection(true);
-        conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
-        conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+        conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
+        conf.setTlsRequireTrustedClientCertOnConnect(true);
+        conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
+        conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
         conf.setNumExecutorThreadPoolSize(5);
+        // Not in use, and because TLS is not configured, it will fail to start
+        conf.setSystemTopicEnabled(false);
         stopBroker();
         startBroker();
         pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
@@ -494,18 +488,8 @@ public void testWebserviceServiceTls() throws Exception {
         final String lookupResourceUrl = "/lookup/v2/topic/persistent/my-property/my-ns/my-topic1";
 
         // set client cert_key file
-        KeyManager[] keyManagers = null;
-        Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
-        PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
-        KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
-        ks.load(null, null);
-        ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
-        KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-        kmf.init(ks, "".toCharArray());
-        keyManagers = kmf.getKeyManagers();
-        TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
-        SSLContext sslCtx = SSLContext.getInstance("TLS");
-        sslCtx.init(keyManagers, trustManagers, new SecureRandom());
+        SSLContext sslCtx = SecurityUtility.createSslContext(false, CA_CERT_FILE_PATH,
+                getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"), "");
         HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());
 
         // hit broker2 url
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
index 246d980d6178c..1e8b26beee38a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
@@ -50,6 +50,7 @@
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig;
 import org.apache.pulsar.functions.sink.PulsarSink;
 import org.apache.pulsar.functions.worker.service.WorkerServiceLoader;
+import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
@@ -62,11 +63,16 @@ public class PulsarFunctionTlsTest {
 
     protected static final int BROKER_COUNT = 2;
 
-    private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
-    private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
-    private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
-    private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
-    private static final String CA_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
+    private final String TLS_SERVER_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
+    private final String TLS_SERVER_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
+    private final String TLS_CLIENT_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
+    private final String TLS_CLIENT_KEY_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
+    private final String CA_CERT_FILE_PATH =
+            ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
 
     LocalBookkeeperEnsemble bkEnsemble;
     protected PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT];
diff --git a/pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem b/pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem
deleted file mode 100644
index 8d0a02f24214f..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/broker-cert.pem
+++ /dev/null
@@ -1,117 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 4098 (0x1002)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Validity
-            Not Before: Feb 17 17:00:44 2021 GMT
-            Not After : Feb 12 17:00:44 2041 GMT
-        Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=localhost/emailAddress=dev@pulsar.apache.org
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:9b:2a:6f:24:02:23:f7:ff:e6:75:61:ca:07:a8:
-                    c0:ab:e9:8d:eb:51:2e:64:f7:9e:9b:d4:b4:be:3a:
-                    fa:f4:6e:c6:92:8f:38:4d:08:cd:89:15:3e:2c:c4:
-                    99:6d:cb:58:80:fc:e0:4d:d6:7d:f6:82:ab:0d:94:
-                    f2:e2:45:c9:d3:15:95:57:0a:6c:86:dc:78:64:3b:
-                    34:4b:01:7c:5d:de:4f:d4:21:1a:5d:27:a0:a5:70:
-                    7a:2e:02:50:e1:19:b4:b9:05:df:99:0d:8b:cc:62:
-                    dc:10:73:fa:72:8b:38:7f:d3:56:54:61:50:bb:92:
-                    ff:09:71:09:c7:bd:04:43:3c:8c:9c:8b:32:d1:05:
-                    04:8a:c6:89:d8:78:56:4d:da:2f:f4:ec:34:37:26:
-                    b5:87:e4:3f:26:c9:41:60:ba:31:10:19:be:f8:0c:
-                    a4:0a:85:19:59:e2:00:5d:b7:c0:bd:d1:2e:fc:a6:
-                    34:8b:85:2a:cc:05:f6:fb:e4:00:e6:74:95:ff:02:
-                    6f:43:7f:39:a7:c2:83:8e:5b:38:40:c9:42:c8:bc:
-                    26:72:36:35:64:c2:54:22:11:87:e8:65:8f:3d:e9:
-                    41:a7:6d:19:88:9a:20:9b:9a:52:e7:d2:cb:b3:e0:
-                    2e:8f:c1:56:54:bc:6d:14:30:73:c5:d7:8e:d0:5a:
-                    5e:cd
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Basic Constraints:
-                CA:FALSE
-            Netscape Cert Type:
-                SSL Server
-            Netscape Comment:
-                OpenSSL Generated Server Certificate
-            X509v3 Subject Key Identifier:
-                49:3C:B2:98:30:CE:7F:79:7A:C6:8B:57:CA:24:9F:12:82:1E:5D:EF
-            X509v3 Authority Key Identifier:
-                keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71
-                DirName:/C=US/ST=California/L=Palo Alto/O=Apache Software Foundation/OU=Pulsar/CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-                serial:52:7B:B4:00:96:60:B4:26:85:BE:01:82:B8:B8:E2:8C:72:EF:5B:90
-
-            X509v3 Key Usage: critical
-                Digital Signature, Key Encipherment
-            X509v3 Extended Key Usage:
-                TLS Web Server Authentication
-    Signature Algorithm: sha256WithRSAEncryption
-         0f:bd:af:39:0c:2c:dc:8f:7e:06:0d:27:df:35:c7:8d:5a:03:
-         68:97:f6:dc:d6:d3:39:0e:b4:76:48:7d:e1:1c:a9:4b:83:fa:
-         52:00:ab:28:93:2d:06:76:0c:14:35:3c:f1:8e:3b:af:c8:d0:
-         27:1f:58:d4:71:22:5f:05:a6:9e:73:c6:a5:5e:2a:e6:fb:eb:
-         fc:73:52:87:ca:8a:2a:f9:1e:5f:e2:b9:bd:01:27:9f:7c:61:
-         a6:97:ad:a0:ab:4e:fb:cc:fa:c8:77:6a:65:1b:ae:60:5e:fb:
-         97:14:8c:40:d7:96:c6:2c:64:59:c0:52:52:7c:2d:98:4b:f4:
-         72:da:83:f7:c6:4f:32:42:ce:df:02:dd:5f:eb:58:42:f9:62:
-         a1:9a:05:ef:13:48:27:af:a3:7f:23:eb:e0:dc:1d:8f:96:2a:
-         88:47:f7:e4:75:6f:a9:15:f6:44:f1:6d:39:3a:2c:df:a7:82:
-         cc:7e:aa:9c:1c:c0:a7:7d:68:31:4a:4e:21:b8:9f:17:90:4b:
-         f1:68:23:ef:a7:53:fc:a9:a8:35:6b:8f:4c:5e:d4:ea:b0:8a:
-         27:9a:86:89:ce:f2:5d:03:35:80:fc:45:e8:87:66:0f:32:b5:
-         2a:f5:1b:79:0e:09:8b:90:40:20:fb:e3:27:8a:c9:92:c1:53:
-         97:10:5a:8c:50:ef:02:46:7e:ec:68:c8:1e:26:66:0e:1d:d6:
-         6c:82:e7:38:14:e8:cb:45:77:29:5f:2c:1a:9d:d7:54:21:8a:
-         cf:0f:b7:0c:ae:fe:d6:fb:fb:c3:07:3e:33:df:59:25:1c:73:
-         d4:87:73:14:b4:76:16:8a:3f:82:05:7b:42:0a:55:0c:79:24:
-         3c:58:31:3f:e0:3e:9f:4e:d0:0e:fd:77:b7:13:2c:d3:d0:46:
-         cc:80:09:0f:50:56:8b:6e:6e:91:b2:5b:c8:2f:4d:86:dc:72:
-         00:de:08:0d:5e:3e:96:1f:12:7d:3b:0d:4d:71:d5:c8:a8:06:
-         ba:00:23:ec:10:4c:a4:c3:6f:bc:f0:d7:b1:cf:57:3f:3b:79:
-         db:80:87:35:c7:4e:7f:bb:38:30:0a:9f:fe:5a:86:f5:97:ce:
-         24:38:79:fd:a0:dc:0b:82:11:a1:ea:0c:e9:16:65:e0:c0:54:
-         80:ad:6e:55:18:ac:27:35:3a:b0:20:70:62:8e:5d:a2:33:53:
-         8c:ce:f9:ee:a1:27:cb:db:e5:9a:5e:e6:f7:80:93:84:63:04:
-         26:58:ab:23:bb:94:80:d0:a0:55:a2:8a:ed:bc:0f:c3:41:d2:
-         26:a5:b9:8d:8a:45:e8:a1:fc:e8:ee:7a:64:93:ed:d6:ef:a2:
-         51:d7:c9:0a:31:39:35:4a
------BEGIN CERTIFICATE-----
-MIIGPDCCBCSgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwgaYxCzAJBgNVBAYTAlVT
-MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNV
-BAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIx
-EjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5h
-cGFjaGUub3JnMB4XDTIxMDIxNzE3MDA0NFoXDTQxMDIxMjE3MDA0NFowgZIxCzAJ
-BgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMSMwIQYDVQQKDBpBcGFjaGUg
-U29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQDDAls
-b2NhbGhvc3QxJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJsqbyQCI/f/5nVhygeowKvp
-jetRLmT3npvUtL46+vRuxpKPOE0IzYkVPizEmW3LWID84E3WffaCqw2U8uJFydMV
-lVcKbIbceGQ7NEsBfF3eT9QhGl0noKVwei4CUOEZtLkF35kNi8xi3BBz+nKLOH/T
-VlRhULuS/wlxCce9BEM8jJyLMtEFBIrGidh4Vk3aL/TsNDcmtYfkPybJQWC6MRAZ
-vvgMpAqFGVniAF23wL3RLvymNIuFKswF9vvkAOZ0lf8Cb0N/OafCg45bOEDJQsi8
-JnI2NWTCVCIRh+hljz3pQadtGYiaIJuaUufSy7PgLo/BVlS8bRQwc8XXjtBaXs0C
-AwEAAaOCAYQwggGAMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCG
-SAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUw
-HQYDVR0OBBYEFEk8spgwzn95esaLV8oknxKCHl3vMIHmBgNVHSMEgd4wgduAFNKy
-PbGkfEhLNuGn3tj8upK6p8RxoYGspIGpMIGmMQswCQYDVQQGEwJVUzETMBEGA1UE
-CAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpBcGFj
-aGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQD
-DAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9y
-Z4IUUnu0AJZgtCaFvgGCuLjijHLvW5AwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQAPva85DCzcj34GDSffNceN
-WgNol/bc1tM5DrR2SH3hHKlLg/pSAKsoky0GdgwUNTzxjjuvyNAnH1jUcSJfBaae
-c8alXirm++v8c1KHyooq+R5f4rm9ASeffGGml62gq077zPrId2plG65gXvuXFIxA
-15bGLGRZwFJSfC2YS/Ry2oP3xk8yQs7fAt1f61hC+WKhmgXvE0gnr6N/I+vg3B2P
-liqIR/fkdW+pFfZE8W05Oizfp4LMfqqcHMCnfWgxSk4huJ8XkEvxaCPvp1P8qag1
-a49MXtTqsIonmoaJzvJdAzWA/EXoh2YPMrUq9Rt5DgmLkEAg++MnismSwVOXEFqM
-UO8CRn7saMgeJmYOHdZsguc4FOjLRXcpXywanddUIYrPD7cMrv7W+/vDBz4z31kl
-HHPUh3MUtHYWij+CBXtCClUMeSQ8WDE/4D6fTtAO/Xe3EyzT0EbMgAkPUFaLbm6R
-slvIL02G3HIA3ggNXj6WHxJ9Ow1NcdXIqAa6ACPsEEykw2+88Nexz1c/O3nbgIc1
-x05/uzgwCp/+Wob1l84kOHn9oNwLghGh6gzpFmXgwFSArW5VGKwnNTqwIHBijl2i
-M1OMzvnuoSfL2+WaXub3gJOEYwQmWKsju5SA0KBVoortvA/DQdImpbmNikXoofzo
-7npkk+3W76JR18kKMTk1Sg==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/broker-key.pem b/pulsar-broker/src/test/resources/authentication/tls/broker-key.pem
deleted file mode 100644
index ee03e754beee1..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/broker-key.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEAmypvJAIj9//mdWHKB6jAq+mN61EuZPeem9S0vjr69G7Gko84
-TQjNiRU+LMSZbctYgPzgTdZ99oKrDZTy4kXJ0xWVVwpshtx4ZDs0SwF8Xd5P1CEa
-XSegpXB6LgJQ4Rm0uQXfmQ2LzGLcEHP6cos4f9NWVGFQu5L/CXEJx70EQzyMnIsy
-0QUEisaJ2HhWTdov9Ow0Nya1h+Q/JslBYLoxEBm++AykCoUZWeIAXbfAvdEu/KY0
-i4UqzAX2++QA5nSV/wJvQ385p8KDjls4QMlCyLwmcjY1ZMJUIhGH6GWPPelBp20Z
-iJogm5pS59LLs+Auj8FWVLxtFDBzxdeO0FpezQIDAQABAoIBAG9pk63mP49l1kM4
-eQjw2Y9WvslVXBuxVNiNbU4eKW1zUO+RGJrvlC027JLWg1g7pwvPBvu85GspPcsd
-xRxFgfonyDhcSrq2+Vb2z8B/i54W73jgX/69YnMIBSKeFRbcD1C+7+MEv/l8jojd
-zdmLL4FQ7O7fhUl57dgIqz4Y8UOYyyBsPpz3pzJLFEb5rE/ajqmFzyl+dO+8140B
-niQ0+7+tAK0njX8OC0WN844GkO24WPCfWhUFrYGkfLq498eRUCWM2YP2tAJ+Uxnh
-v3K9icDwOX6PJXYlbvNEUCE+t60NoDYHcMpfzUdFEhBYpKadfKE/RFFcu0vAZ+aR
-y24oAuECgYEAyPLYXWIs88pPHQhSf2DAMRref5eeV+XA6Dy/P+z8z0bA7I6X9dl6
-AK6rRKGJl9HI7c/Gky6P10fymopYopNkClXm7SBTLKx0vfjil0U6Mx5ZsfDspE3q
-0o9MJKVgobCxVZlLErU55XzktKwjlv2UvDX7VuxRndqN9qdf+YSMb9kCgYEAxayx
-sOrJcPZVfy3Ohy5CeStF+E2dtfcKB7M7xZxZqykVy+6J1XjXHmp1L7Wpi0ju57Hi
-l2ZqKasHDwtlLOnfSTbvC47hsa1ydnoFTjJBObR1wS43oVkyV0AHid4w81ddOWPC
-H0ZmhvNe7pUxm5crpxsY6hAAraJ4Hej23MOxghUCgYEAip26UvCeQa2U1VogTm3X
-Jgh641kbiVabs5fz9Yzs966+9m+Gs7jJSB81Vap415mHGUTyniTIZKDk4WX9rmgt
-4lNPcNOTjIWKImHFLMQ8WXbeOLkRBGYbThQ7WiwadG8GZR3Rg54vyfZVbawxAL78
-ErjKIDP0OQfCVhsvQVgF6EECgYEAlQ2P+xA/Dv+gHkLjDUmTdBxuKToVZqU9merL
-cklfz9EuD1Tx99ajltq9PFll25IGGw0mB/WAraS5sN1tz/0VkfZrL7LwefKIcc+2
-em0og6OQezcnWXGRpPqx9IJnNMY2lFSlhsGmA7I1bf9vpZvKnbmwAqZIbKUqn5sP
-sg2ZprUCgYEApAVD+9wXfZE/YDHVZX1k6p38ORqjq/04AJkL/LmUW5DL5to1+1KQ
-Q438HzMtYIq7aZyzWmlF6DmyN5mxKKK3yY79p0rvdV74AoT+ucDzM3ge0Md7liCs
-0GwNnDSiPzdau738UoIKc1VbF7dMDL3LzqnfrBUCr7nXRbR3BHHuqws=
------END RSA PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/cacert.pem b/pulsar-broker/src/test/resources/authentication/tls/cacert.pem
deleted file mode 100644
index 6abfc2d80c123..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/cacert.pem
+++ /dev/null
@@ -1,127 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number:
-            52:7b:b4:00:96:60:b4:26:85:be:01:82:b8:b8:e2:8c:72:ef:5b:90
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Validity
-            Not Before: Feb 17 16:43:44 2021 GMT
-            Not After : Feb 12 16:43:44 2041 GMT
-        Subject: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (4096 bit)
-                Modulus:
-                    00:b1:3c:7d:ab:4a:54:72:37:2a:92:94:0a:66:46:
-                    af:8c:ed:f4:2e:f3:87:1a:d0:c7:9d:23:35:1b:61:
-                    74:69:ca:f7:f5:3e:95:9c:86:f2:21:34:f8:0b:ed:
-                    45:76:22:ec:75:52:c0:67:db:2f:ba:da:25:3f:e1:
-                    5b:ac:da:15:dd:a5:75:24:b2:12:f0:b0:ce:fd:ab:
-                    44:06:a9:09:f6:b0:8e:8f:83:53:16:69:fa:9c:cc:
-                    00:fa:dd:13:f3:da:fd:f2:bf:88:8e:c4:f8:1a:6f:
-                    ab:4d:f8:32:81:80:7e:51:7a:99:2d:94:cd:f3:5d:
-                    1c:58:b2:44:f1:96:12:46:56:bd:60:8f:65:32:b7:
-                    d4:4b:7b:f3:23:88:2d:9b:a4:c4:c9:52:ea:9f:66:
-                    c1:74:be:4b:91:c6:b9:57:ec:c1:cc:81:bb:03:d5:
-                    fa:a0:46:4f:9a:a7:3e:3c:27:26:2b:97:eb:69:53:
-                    04:75:50:97:d6:0d:90:b1:37:9f:64:df:70:4d:d9:
-                    b3:e3:b7:cc:76:50:d9:3c:9b:4c:ac:e9:26:2e:cf:
-                    ac:47:42:14:b7:60:00:0a:de:42:47:66:0c:c7:7a:
-                    b9:4d:f4:fb:c2:6a:45:78:ec:b0:b4:ce:b3:1f:50:
-                    25:96:13:0c:55:0a:e0:d6:76:f7:1f:e1:16:e6:41:
-                    d6:72:6a:49:17:12:d9:05:8f:dc:56:b6:31:b3:b7:
-                    9c:e3:d8:a9:99:8a:1d:3b:9d:d9:59:44:ee:46:88:
-                    11:5f:ab:fa:38:a9:8b:d2:23:15:8b:af:1a:de:66:
-                    ba:7d:51:95:37:94:91:aa:01:01:d7:83:19:4b:5d:
-                    8d:f4:18:39:ef:e3:32:d0:62:c8:12:50:4e:91:c2:
-                    ac:58:73:68:bb:92:20:fc:14:e5:1a:86:bd:40:4c:
-                    94:e0:7d:0d:9c:08:57:ae:00:44:38:94:a3:3d:64:
-                    99:43:f8:e3:12:90:14:0f:5d:63:e2:c6:07:ea:d0:
-                    4c:8e:cf:e0:ae:34:be:86:4f:fc:58:e2:ea:f5:23:
-                    82:37:96:02:57:1b:b4:29:ca:fd:68:a0:48:79:e8:
-                    31:97:9a:5a:0e:2b:b4:b0:84:bb:57:4e:5f:4f:a7:
-                    43:45:97:d7:de:05:fc:2f:6c:3e:f5:53:26:56:a3:
-                    a5:da:52:69:57:8e:a0:4b:27:50:f9:ad:6e:76:a6:
-                    29:cc:06:94:dd:d0:ac:c6:18:22:a0:e2:bb:ed:d5:
-                    e4:97:f7:ac:23:df:75:30:41:97:07:3f:d3:12:8e:
-                    c5:a4:ef:ce:40:e8:3b:57:24:19:33:1b:ee:8a:0e:
-                    dd:0c:70:f2:1a:87:35:d9:71:d8:18:a7:9c:47:db:
-                    93:51:c3
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Subject Key Identifier:
-                D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71
-            X509v3 Authority Key Identifier:
-                keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71
-
-            X509v3 Basic Constraints: critical
-                CA:TRUE
-            X509v3 Key Usage: critical
-                Digital Signature, Certificate Sign, CRL Sign
-    Signature Algorithm: sha256WithRSAEncryption
-         14:3d:7c:15:86:de:aa:5a:30:5d:d4:f2:bc:5f:10:d2:af:fe:
-         91:d7:ee:f3:b8:5f:ce:e4:c9:b2:01:c3:16:da:66:8e:7e:b1:
-         c1:e3:30:ff:1d:73:d0:9c:20:3d:54:32:57:ae:07:80:4a:24:
-         6e:7e:32:a3:e7:23:4d:5c:31:54:8b:c1:1b:c5:bc:20:5d:43:
-         62:93:e0:2e:a7:01:77:39:cf:fd:ec:4c:57:09:4f:2b:ad:ac:
-         b6:c0:be:5a:a3:ea:12:ac:5a:7f:60:23:81:bb:9a:fa:5f:7a:
-         67:a9:31:c3:34:af:db:ff:32:22:83:40:c2:7d:2f:39:5e:8a:
-         29:44:73:5f:6e:b4:f4:a2:ae:60:1f:8e:ef:91:9a:49:bb:a6:
-         90:2b:e0:44:95:24:8b:37:90:18:2d:41:32:8a:8e:07:8d:ea:
-         75:62:b8:9c:ec:73:6f:12:54:23:6d:40:00:74:c7:d3:fb:b7:
-         95:06:7d:cc:6d:8e:2c:d0:8b:11:06:8a:b7:43:1a:d7:e9:98:
-         f4:c6:ef:ad:2a:75:08:fb:07:8f:20:36:7a:86:1a:cf:f7:d6:
-         96:ad:ed:71:59:d1:81:56:18:8d:98:c2:c0:44:e5:29:7a:7c:
-         c0:e3:d7:fb:b8:f5:b2:50:53:8a:cf:38:ff:99:aa:bb:28:51:
-         60:e8:05:91:e1:ee:86:90:90:9b:87:60:63:38:cf:54:a5:82:
-         74:0f:40:b5:d2:6a:c5:a9:98:22:59:4e:fb:a5:81:e2:7b:0e:
-         3f:71:f3:24:17:1e:c5:89:fc:ae:ed:f3:69:65:02:b8:1e:98:
-         bc:37:c6:25:36:f8:ca:99:60:8e:13:3b:33:ec:91:b3:eb:04:
-         6d:41:97:3e:35:c0:97:ed:66:12:25:44:23:f3:2e:fa:9c:2e:
-         c2:ba:dd:f3:63:d7:5b:b2:72:03:4d:3b:fb:5e:29:d6:5c:02:
-         32:93:47:d1:4c:77:4a:58:c5:aa:81:ab:67:84:80:81:14:28:
-         e1:db:11:16:6d:31:50:7a:47:b2:a8:2d:15:a1:c4:63:1b:ce:
-         d5:e1:d7:57:dc:1a:71:e0:55:9f:6d:fb:be:e6:99:e8:89:be:
-         2c:e0:19:5e:cd:02:79:52:ee:93:56:9f:dc:d7:de:31:9b:2a:
-         c8:91:48:a0:c7:44:7d:72:32:27:c3:2b:d8:e8:6b:94:67:b5:
-         1d:9d:99:25:23:d9:24:b5:ed:4b:f2:18:2d:88:f5:d4:36:bb:
-         53:8c:a8:b1:7f:05:13:d7:8d:89:9d:55:33:90:bc:60:99:cf:
-         05:ba:bd:cb:c5:61:f9:c5:1a:f7:46:9c:40:90:dd:83:aa:7a:
-         1f:ab:5c:10:8d:26:27:1e
------BEGIN CERTIFICATE-----
-MIIGPzCCBCegAwIBAgIUUnu0AJZgtCaFvgGCuLjijHLvW5AwDQYJKoZIhvcNAQEL
-BQAwgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQH
-DAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9u
-MQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3
-DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMB4XDTIxMDIxNzE2NDM0NFoXDTQx
-MDIxMjE2NDM0NFowgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh
-MRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBG
-b3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEk
-MCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMIICIjANBgkqhkiG
-9w0BAQEFAAOCAg8AMIICCgKCAgEAsTx9q0pUcjcqkpQKZkavjO30LvOHGtDHnSM1
-G2F0acr39T6VnIbyITT4C+1FdiLsdVLAZ9svutolP+FbrNoV3aV1JLIS8LDO/atE
-BqkJ9rCOj4NTFmn6nMwA+t0T89r98r+IjsT4Gm+rTfgygYB+UXqZLZTN810cWLJE
-8ZYSRla9YI9lMrfUS3vzI4gtm6TEyVLqn2bBdL5Lkca5V+zBzIG7A9X6oEZPmqc+
-PCcmK5fraVMEdVCX1g2QsTefZN9wTdmz47fMdlDZPJtMrOkmLs+sR0IUt2AACt5C
-R2YMx3q5TfT7wmpFeOywtM6zH1AllhMMVQrg1nb3H+EW5kHWcmpJFxLZBY/cVrYx
-s7ec49ipmYodO53ZWUTuRogRX6v6OKmL0iMVi68a3ma6fVGVN5SRqgEB14MZS12N
-9Bg57+My0GLIElBOkcKsWHNou5Ig/BTlGoa9QEyU4H0NnAhXrgBEOJSjPWSZQ/jj
-EpAUD11j4sYH6tBMjs/grjS+hk/8WOLq9SOCN5YCVxu0Kcr9aKBIeegxl5paDiu0
-sIS7V05fT6dDRZfX3gX8L2w+9VMmVqOl2lJpV46gSydQ+a1udqYpzAaU3dCsxhgi
-oOK77dXkl/esI991MEGXBz/TEo7FpO/OQOg7VyQZMxvuig7dDHDyGoc12XHYGKec
-R9uTUcMCAwEAAaNjMGEwHQYDVR0OBBYEFNKyPbGkfEhLNuGn3tj8upK6p8RxMB8G
-A1UdIwQYMBaAFNKyPbGkfEhLNuGn3tj8upK6p8RxMA8GA1UdEwEB/wQFMAMBAf8w
-DgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQAUPXwVht6qWjBd1PK8
-XxDSr/6R1+7zuF/O5MmyAcMW2maOfrHB4zD/HXPQnCA9VDJXrgeASiRufjKj5yNN
-XDFUi8EbxbwgXUNik+AupwF3Oc/97ExXCU8rray2wL5ao+oSrFp/YCOBu5r6X3pn
-qTHDNK/b/zIig0DCfS85XoopRHNfbrT0oq5gH47vkZpJu6aQK+BElSSLN5AYLUEy
-io4Hjep1Yric7HNvElQjbUAAdMfT+7eVBn3MbY4s0IsRBoq3QxrX6Zj0xu+tKnUI
-+wePIDZ6hhrP99aWre1xWdGBVhiNmMLAROUpenzA49f7uPWyUFOKzzj/maq7KFFg
-6AWR4e6GkJCbh2BjOM9UpYJ0D0C10mrFqZgiWU77pYHiew4/cfMkFx7Fifyu7fNp
-ZQK4Hpi8N8YlNvjKmWCOEzsz7JGz6wRtQZc+NcCX7WYSJUQj8y76nC7Cut3zY9db
-snIDTTv7XinWXAIyk0fRTHdKWMWqgatnhICBFCjh2xEWbTFQekeyqC0VocRjG87V
-4ddX3Bpx4FWfbfu+5pnoib4s4BlezQJ5Uu6TVp/c194xmyrIkUigx0R9cjInwyvY
-6GuUZ7UdnZklI9kkte1L8hgtiPXUNrtTjKixfwUT142JnVUzkLxgmc8Fur3LxWH5
-xRr3RpxAkN2Dqnofq1wQjSYnHg==
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/client-cert.pem b/pulsar-broker/src/test/resources/authentication/tls/client-cert.pem
deleted file mode 100644
index 45f3cde215fe5..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/client-cert.pem
+++ /dev/null
@@ -1,90 +0,0 @@
-Certificate:
-    Data:
-        Version: 1 (0x0)
-        Serial Number: 4097 (0x1001)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org
-        Validity
-            Not Before: Feb 17 16:56:55 2021 GMT
-            Not After : Feb 12 16:56:55 2041 GMT
-        Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=admin/emailAddress=dev@pulsar.apache.org
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:ab:61:f5:12:b1:e1:ae:19:01:3e:59:4a:c6:ca:
-                    00:0c:96:e8:76:3a:83:20:d9:af:3a:e1:11:20:12:
-                    e0:e4:d0:70:8f:4b:7b:af:e1:89:ef:9b:c5:a9:c2:
-                    ed:ae:24:8d:bb:42:6e:ec:59:11:3f:f5:63:59:61:
-                    18:9f:70:b6:76:88:e2:ca:79:15:cc:fb:9c:5e:5c:
-                    bb:a1:d7:f0:d8:11:d4:17:34:1e:81:7e:0b:0e:05:
-                    be:5d:fa:d6:46:af:e1:95:d8:a0:5d:c5:2f:d9:a9:
-                    8f:69:64:49:95:f7:42:16:6a:84:2b:2e:af:91:73:
-                    3d:b6:d4:44:56:9a:61:43:49:15:22:ae:90:5d:04:
-                    29:90:4e:b2:41:34:73:3e:a2:48:05:1c:bc:8e:1b:
-                    0b:c1:d5:df:56:32:40:e9:91:a2:7b:de:31:2b:67:
-                    f1:8e:d6:c5:c0:87:57:70:29:f9:af:db:57:a0:2e:
-                    8c:30:0a:a7:47:39:33:4c:d7:2d:32:aa:48:29:bd:
-                    c4:48:c5:58:52:07:c4:99:b1:cc:66:da:ac:28:4d:
-                    c1:bc:1f:44:3f:a3:63:61:bd:ff:48:61:76:04:b2:
-                    7d:1c:6e:9c:ee:82:bb:f7:60:1c:7a:a0:98:be:2d:
-                    70:43:2f:64:bf:d2:0f:20:25:f7:c7:7d:70:05:b8:
-                    2e:bf
-                Exponent: 65537 (0x10001)
-    Signature Algorithm: sha256WithRSAEncryption
-         1c:31:b8:0f:a1:03:28:a0:da:31:ec:34:ce:e0:fd:01:99:9d:
-         9b:ad:f8:03:5d:20:85:18:de:ca:b5:ea:61:c9:3b:65:42:9c:
-         e5:21:73:d2:06:41:4b:a9:3a:fb:7f:ff:45:f3:5a:4a:ab:5a:
-         86:cd:57:6a:5f:13:c0:ae:7e:ad:5c:6e:c3:c4:e7:b7:d3:14:
-         bf:86:fe:f2:d1:70:0e:fc:98:50:a7:fe:53:62:5a:2d:f5:63:
-         2c:ee:4a:7c:dd:32:3e:d1:52:3a:1f:15:38:4b:2a:4a:ee:27:
-         a9:d8:92:a8:33:92:83:c9:3a:09:5a:01:66:0e:68:da:8f:82:
-         c0:18:cc:78:ea:c5:db:09:7c:2f:61:c3:51:f8:58:7a:27:d7:
-         92:c0:ff:f8:29:d7:a0:e9:54:17:8d:48:a8:ff:5e:92:ee:81:
-         6c:37:90:1c:93:28:8c:d2:f5:b1:20:96:d3:1d:0f:c0:7f:db:
-         0c:6d:65:7f:3a:55:e5:c9:9a:ad:09:91:a5:57:cb:fc:bf:df:
-         69:bd:6b:87:94:5b:d0:cf:3b:8b:48:41:3d:56:b6:1d:3f:e7:
-         f6:b6:58:f7:54:2a:dd:da:60:68:db:9b:70:04:8b:19:c3:44:
-         bf:1d:b4:28:b9:f8:ea:ad:d3:1a:6e:64:72:b1:61:6a:f3:e1:
-         d4:68:56:7b:0e:ad:4c:53:1e:d2:2e:1c:bc:b7:82:59:af:65:
-         d2:fd:ef:89:7c:34:8f:51:a1:4e:9d:7e:dc:c7:97:68:ea:aa:
-         e5:67:ed:be:dc:38:74:0e:c3:6f:fd:08:62:54:d8:1f:15:d1:
-         25:fc:21:f6:8c:f9:2f:65:5e:07:b9:e9:56:ba:48:14:5c:0d:
-         18:ba:f8:83:54:5b:b6:27:0c:36:2c:20:29:9c:c2:68:c5:3a:
-         0f:a5:d6:5f:7c:aa:f9:a6:2a:2b:69:c5:b1:39:e7:1c:02:31:
-         5b:f5:82:de:c9:4e:8d:33:dc:94:02:44:0a:44:95:75:7b:a1:
-         e7:ee:92:fc:35:93:73:8c:22:c1:32:ea:39:17:ca:d0:87:fc:
-         4d:8e:04:f8:59:66:d3:14:3f:59:ad:76:14:20:16:7b:77:4f:
-         94:58:f8:85:5c:ba:b3:69:ed:7f:75:54:9a:1a:88:21:5d:04:
-         57:87:85:e2:d4:0e:1b:61:7f:5d:36:dc:72:a1:9d:0b:c8:ce:
-         19:69:49:fa:1b:bb:3f:3d:1b:4d:81:42:95:4e:d8:0b:04:d1:
-         08:6d:15:b3:ae:52:41:12:ff:e1:90:c4:7d:52:88:55:8b:87:
-         83:06:48:8b:fc:3a:a7:47:0e:6c:a8:4c:9e:b0:aa:da:50:f5:
-         97:97:98:3e:9d:18:ef:43
------BEGIN CERTIFICATE-----
-MIIEqzCCApMCAhABMA0GCSqGSIb3DQEBCwUAMIGmMQswCQYDVQQGEwJVUzETMBEG
-A1UECAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpB
-cGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYD
-VQQDDAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hl
-Lm9yZzAeFw0yMTAyMTcxNjU2NTVaFw00MTAyMTIxNjU2NTVaMIGOMQswCQYDVQQG
-EwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEjMCEGA1UECgwaQXBhY2hlIFNvZnR3
-YXJlIEZvdW5kYXRpb24xDzANBgNVBAsMBlB1bHNhcjEOMAwGA1UEAwwFYWRtaW4x
-JDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCCASIwDQYJKoZI
-hvcNAQEBBQADggEPADCCAQoCggEBAKth9RKx4a4ZAT5ZSsbKAAyW6HY6gyDZrzrh
-ESAS4OTQcI9Le6/hie+bxanC7a4kjbtCbuxZET/1Y1lhGJ9wtnaI4sp5Fcz7nF5c
-u6HX8NgR1Bc0HoF+Cw4Fvl361kav4ZXYoF3FL9mpj2lkSZX3QhZqhCsur5FzPbbU
-RFaaYUNJFSKukF0EKZBOskE0cz6iSAUcvI4bC8HV31YyQOmRonveMStn8Y7WxcCH
-V3Ap+a/bV6AujDAKp0c5M0zXLTKqSCm9xEjFWFIHxJmxzGbarChNwbwfRD+jY2G9
-/0hhdgSyfRxunO6Cu/dgHHqgmL4tcEMvZL/SDyAl98d9cAW4Lr8CAwEAATANBgkq
-hkiG9w0BAQsFAAOCAgEAHDG4D6EDKKDaMew0zuD9AZmdm634A10ghRjeyrXqYck7
-ZUKc5SFz0gZBS6k6+3//RfNaSqtahs1Xal8TwK5+rVxuw8Tnt9MUv4b+8tFwDvyY
-UKf+U2JaLfVjLO5KfN0yPtFSOh8VOEsqSu4nqdiSqDOSg8k6CVoBZg5o2o+CwBjM
-eOrF2wl8L2HDUfhYeifXksD/+CnXoOlUF41IqP9eku6BbDeQHJMojNL1sSCW0x0P
-wH/bDG1lfzpV5cmarQmRpVfL/L/fab1rh5Rb0M87i0hBPVa2HT/n9rZY91Qq3dpg
-aNubcASLGcNEvx20KLn46q3TGm5kcrFhavPh1GhWew6tTFMe0i4cvLeCWa9l0v3v
-iXw0j1GhTp1+3MeXaOqq5Wftvtw4dA7Db/0IYlTYHxXRJfwh9oz5L2VeB7npVrpI
-FFwNGLr4g1RbticMNiwgKZzCaMU6D6XWX3yq+aYqK2nFsTnnHAIxW/WC3slOjTPc
-lAJECkSVdXuh5+6S/DWTc4wiwTLqORfK0If8TY4E+Flm0xQ/Wa12FCAWe3dPlFj4
-hVy6s2ntf3VUmhqIIV0EV4eF4tQOG2F/XTbccqGdC8jOGWlJ+hu7Pz0bTYFClU7Y
-CwTRCG0Vs65SQRL/4ZDEfVKIVYuHgwZIi/w6p0cObKhMnrCq2lD1l5eYPp0Y70M=
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/authentication/tls/client-key.pem b/pulsar-broker/src/test/resources/authentication/tls/client-key.pem
deleted file mode 100644
index e12697c966a9c..0000000000000
--- a/pulsar-broker/src/test/resources/authentication/tls/client-key.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEoQIBAAKCAQEAq2H1ErHhrhkBPllKxsoADJbodjqDINmvOuERIBLg5NBwj0t7
-r+GJ75vFqcLtriSNu0Ju7FkRP/VjWWEYn3C2dojiynkVzPucXly7odfw2BHUFzQe
-gX4LDgW+XfrWRq/hldigXcUv2amPaWRJlfdCFmqEKy6vkXM9ttREVpphQ0kVIq6Q
-XQQpkE6yQTRzPqJIBRy8jhsLwdXfVjJA6ZGie94xK2fxjtbFwIdXcCn5r9tXoC6M
-MAqnRzkzTNctMqpIKb3ESMVYUgfEmbHMZtqsKE3BvB9EP6NjYb3/SGF2BLJ9HG6c
-7oK792AceqCYvi1wQy9kv9IPICX3x31wBbguvwIDAQABAoIBAQCcwbSPrPRncaeZ
-h8LFoO36le16dnqKCZIloMcxNxNNNvo9lyVC8mBgMXLSm+Eab4TTyyf6Nl14ytJc
-ZltHOqkqMnp+B9LQ8zNLfDaDCijY+TWtI5bjio5B/S7qdwyXCzii/slv+3SQ+m6a
-T4ifCtH//t11QfaEa4v/NphrPjnIeAgB681bk8nKdRop84ar+51lgbHoAza+wv+8
-e+aK3Od8r4yD19ZoPiMg0o4t2cEi8kupVgjsuZVtcvF9Q6QLYV17BFYEHqYjcr18
-N1EJ96f2FLO6cwEM+cG4n8gHjfDGRcDlhT9Cum1kDpg4J88auVUXnrDyi5Dcv1Pz
-6EC+ZmXBAoGBAOHUSUDMkbEePKDaM3Z+4jLqZWc3UZhxQLnqg5l7phdQ6iSogQX9
-1LpZCJ+lOMTHBCnaTCoQpuSHgYgraVkD4KG6nzC423oDesd/xNvlfW3TRsmwZWbL
-khdcdBSoVy3Kbv1v8kxw0NlcR68qo1XYfmFCAITcFHdxDz/jGStydlR9AoGBAMJH
-gyPenL595X8t47R93rkGOIx5cVf5YrDIZCByp4K44Tf9OqZHbky7jSPSSbur10mI
-pypRq5EcZ/cudU4w4gGaMauczt5Dgvlqd3T+GTZY3jO8bxi66gvzYTbigAxaJWcY
-Uafiv5W9ldRKsY3pyCL8ubg38Ed2cSaS2wGd/SDrAn8NO2MPaO0gc6UZx688QjL+
-yL0oTxV42Snxusv7MkOJGjSd8UGeGEFeqdjXgdbRsNeNnDzaOh+NRGNSlziU/qUq
-1MR/FlXF0G5hQhtGxyuSQ87iAnPukf79X21tyG9TP4lBUE3iLLoQAlgw606muQiu
-qi9dmYeZeAZst+HBqfNFAoGADg6qmH/VC5uEbY1eeoLZCL5AfTmUT+9FitEVHZvu
-LvE9qpVyFvH4Mykm7z6aAzBN5Y4zukYqiddqVmJQLpYu5DrJ+UbhWQe9hFqFxjtU
-i7Amc8vgpgNwR+kWUahV547mQe1qiyFHB4iuPKwi6MfPqWhr775sbl9NlKLvodBS
-rn0CgYBDrLH6ehNV/RnJIVZYQD6YcocYdYFy4u76mCYKmEP57XmstZHZXQgiRwbK
-Oy2Yg/qieKtSMjstgHFK6ZNYIR37l9J9Lh9aeal61+wW2dsGEy29Rhg01FpvKReq
-wCHz3tneUyaOhq9m0gKMOpWYcO+FBX1/2K5Gwj8FgEpu9r2b3w==
------END RSA PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/certificate/client.crt b/pulsar-broker/src/test/resources/certificate/client.crt
deleted file mode 100644
index 2d7d156866a86..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/client.crt
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDVjCCAj4CCQCtw/UnTFDT7DANBgkqhkiG9w0BAQUFADBtMQswCQYDVQQGEwJB
-VTETMBEGA1UECAwKU29tZS1TdGF0ZTEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MSEw
-HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMMBmNsaWVu
-dDAeFw0xNjA2MjAwMTQ1NDZaFw0yNjA2MTgwMTQ1NDZaMG0xCzAJBgNVBAYTAkFV
-MRMwEQYDVQQIDApTb21lLVN0YXRlMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxITAf
-BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQV5F3Au9FWXIYPdWqiX
-Rk5gdVmVkDuuFK4ZoOd8inoJpB3PPkpmpgoVkKQHDFhgx3ODGWIUgo+n6QDsJxY4
-ygHfVeggQgek8iUfteYVsIcHS0bjkhIij/3ihC301FkiqbrV069oLvUXLKcv3zxG
-mdBAiz0k4xGZhFieVRvQCLY9syUUxmQ/3Cv42lDY8a1gTw4CRRx/hCfDvXCKhOT4
-bMwUIDZfHB3JoDh3Thp8FLz0nTrRF75mSQJ/OdcafIm0Xoz2Otp/CSxLS+U1lLvG
-05crWTDe0om7NW4mK4CqGCFq5gUw7eIzaeO7Q5Qez9XGTMzkgIDTMvNYGGEeJhhm
-NQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQAKXy4g6hljY5MpO8mbZh+uJHq6NEUs
-4dr7OKDDWc39AROZsGf2eFUmHOjmRSw7VHpguGKI+rFRELVffpg/VvMh5apu+DBf
-jhxtDNceAyh5uugPNUJHXyeikBDYW8bAzUU3DmMldPkTZWcGjurmyhDQ1TtK2YJe
-RMFBXw5aAzdJMNi6OfXDH/ZX32hrb482yghDZj+ndnm0FefmLbFTQRMF8/fIHb1W
-kqNHwIaapZwH6j/MJy/TRFYcJunrBUYT9zVjY46k3GU0ex/Bn7T4pg9gzgFGZJhn
-jQQFKliIC84thCzdlPkrLduLY8tmlDKpLXatbEQ+s1MmNOURm6irPp6g
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/certificate/client.csr b/pulsar-broker/src/test/resources/certificate/client.csr
deleted file mode 100644
index e01f33ef073f6..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/client.csr
+++ /dev/null
@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICsjCCAZoCAQAwbTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
-FTATBgNVBAcMDERlZmF1bHQgQ2l0eTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
-cyBQdHkgTHRkMQ8wDQYDVQQDDAZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IB
-DwAwggEKAoIBAQCpBXkXcC70VZchg91aqJdGTmB1WZWQO64Urhmg53yKegmkHc8+
-SmamChWQpAcMWGDHc4MZYhSCj6fpAOwnFjjKAd9V6CBCB6TyJR+15hWwhwdLRuOS
-EiKP/eKELfTUWSKputXTr2gu9Rcspy/fPEaZ0ECLPSTjEZmEWJ5VG9AItj2zJRTG
-ZD/cK/jaUNjxrWBPDgJFHH+EJ8O9cIqE5PhszBQgNl8cHcmgOHdOGnwUvPSdOtEX
-vmZJAn851xp8ibRejPY62n8JLEtL5TWUu8bTlytZMN7Sibs1biYrgKoYIWrmBTDt
-4jNp47tDlB7P1cZMzOSAgNMy81gYYR4mGGY1AgMBAAGgADANBgkqhkiG9w0BAQUF
-AAOCAQEAk3eueaq/gonBzKH75oWHlqPbMZQFk4NXqx8h24ZfkCzPEFPyDM+jdQxv
-8vDtyWq+fizqAQmGrM7WPHgnTbmZyovfmwuKwtTlkD/8t7XpTmm9fYspbL4WzdP1
-y8/Vug09te+rni+v+kjk5b9IceEy6kLvXuzirE6c4LunAm+thrr5gWmsx1pyDiq7
-W2M15UZrm/paaCg6cVaMFdXCRZP+g1P4NcgDUe2TyFbLlhOJNtX3DJRZWEhrkEYK
-mRz2tJuiuitCzheAgRrFXepRagHKYffNSas1n/2kIc9QpZ8654kxsAzEwL7CnHd/
-SHbMS9dfP+uM6DACwcvngSOBMJ9KMg==
------END CERTIFICATE REQUEST-----
diff --git a/pulsar-broker/src/test/resources/certificate/client.key b/pulsar-broker/src/test/resources/certificate/client.key
deleted file mode 100644
index 34fc701c5257d..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/client.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCpBXkXcC70VZch
-g91aqJdGTmB1WZWQO64Urhmg53yKegmkHc8+SmamChWQpAcMWGDHc4MZYhSCj6fp
-AOwnFjjKAd9V6CBCB6TyJR+15hWwhwdLRuOSEiKP/eKELfTUWSKputXTr2gu9Rcs
-py/fPEaZ0ECLPSTjEZmEWJ5VG9AItj2zJRTGZD/cK/jaUNjxrWBPDgJFHH+EJ8O9
-cIqE5PhszBQgNl8cHcmgOHdOGnwUvPSdOtEXvmZJAn851xp8ibRejPY62n8JLEtL
-5TWUu8bTlytZMN7Sibs1biYrgKoYIWrmBTDt4jNp47tDlB7P1cZMzOSAgNMy81gY
-YR4mGGY1AgMBAAECggEAcJj3yVhvv0/BhY8+CCYl2K1f7u1GCLbpSleNNTbhLbMM
-9yrwo/OWnGg9Y4USOPQrTNOz81X2id+/oSZ/K67PGCvVJ3qi+rny9WkrzdbAfkAF
-6O0Jr4arRbeBjkK7Rjc3M1EHH6VLx3R5AsNBzfpuogss5FVQXICd/5+1oscLeLEx
-/Fn+51IEn9FUg5vr7ElG51f+zPxexcWHLNoqGjTEIGGtI8/CfTzD9tBV4sIjf/Nc
-Zzfs9XYrChfcrS0U1zDa+L7c5gYfoN6M08sBiuZlhyyO9wgzPlp+XnsrSFv6hUta
-0scjAbN4bh+orQn6zgFN/sjkQnraWXW7pKFLyTR/IQKBgQDVju4IbhE9XRweNgXi
-s3BuGV+HsuFffEf0904/zCuCUcScGb5WCz5+KtlFJ//YxfocHVZajH+4GdCGbWim
-m+H3XvRpWgfK/aBNOXu5ueLbnPYyPjTrcpKRsomeoiV+Jz1tv5PQElwzCiCzVvQf
-fMyhQT16YIsFQAGJzQMBEHWODQKBgQDKnKps3sKSR3ycUtIxCVXUir7p52qst0Pm
-bPO8JrcRKZP2z8MJB96+DcQFzrxj7t5DDktkYEsFOPPuIeUsYXsY+MKHs4hEQVCz
-hpDJJNQ8s+SV8TLzKpinZEmLIjslLbn2rQrpqybPg84VxqX3qqM8IrXhMf77aGj6
-QHqvQwHWyQKBgQDF1RVO+9++j82ncvY6z22coKath5leIjxqgtqbISFBJUxUK0j2
-Xo4yxLDnbqmE/8m1V7wSP8tlGYzhquLiTM+kn/Mc0Ukc0503TMQABmJQfXRYkOXn
-IwkCLXltWdoPpnwyeeGNRCTjJ0OpvyiBLtRFobE498xxPZzvMdrRlpS/1QKBgQCo
-wmMleUnBQ2/kWQugMnFeLg6kjs+IesFAnYFKN0kGL4aB7j06OWbrEFY0rCS4bA6O
-9coQGjCCchSjRXI4TB2XCCQnmX8nsuuADNZt45Iv2XrM9XEFn3Y0/tBO5j0zU2nw
-r+NGC/uwns050BMPPf7mqNarctQ6HZZK0wgdEQfoGQKBgC+pbkQv9cn68TsiaJ3w
-tvNRTXCIAAH4Vtn9Cp+63ao+kXn94BJqQF99i58kJpG4ol6wbCHUoC6fHgxUh5HB
-JB0HjC2eCMgn4acAQg0sPW6l35KX36yYxtrL7eosB/yBYum0XAwmboNjEhlCZkOs
-YOpSsn61g7xqqrt40Spb5vUn
------END PRIVATE KEY-----
diff --git a/pulsar-broker/src/test/resources/certificate/server.crt b/pulsar-broker/src/test/resources/certificate/server.crt
deleted file mode 100644
index 59b651be2a406..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/server.crt
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDLjCCAhYCCQDn/Yvym+FMsDANBgkqhkiG9w0BAQUFADBZMQswCQYDVQQGEwJB
-VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0
-cyBQdHkgTHRkMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTYwNjEzMjIyMTQ2WhcN
-MjYwNjExMjIyMTQ2WjBZMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0
-ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRIwEAYDVQQDEwls
-b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCs29IuzZvk
-OGUkS/wqKzd/h2esqjCSjw4SLLbeh1GA3UEvh1k9+eRiYwJG1yCOHmcsp4A8Du99
-8xbgeihpWWw7pjL5VVky3ciuvHyz1Cc6bKRps/GzVJBwFP0gzHnK8bUM86U52yGT
-1DepD/Y2lURy0igdVcAMjGweMwoTmiaVcwZexfYuEef+jz3fmpmOwP9rboIA9rQr
-mTbLzzkbAwZXdl+bRvIefIjIazIzTOs8tJWrhFaTJUgBhhLjFIwTdpS+n+FqOu8J
-92K+PvKjIeJ3kmnZyRHK7uidlAn0g/DK+co1sX3zORPCWeg21K+/vVHTj91zARNb
-O9hVS4bqqsw9AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBACE0WBuTbHcPtYKv2ZMS
-mYk9jvtAhmWHQ6tNqV8CmS2AsrzZdWglGaqIRsm5slkD2BGeQS+BesTArUuENTmP
-r9kJSecdiiB8aWtLbhoCSH3QR6IW/b5UVl6sR5OIh7SkNTjMSUSDnMEVLNGyKZGS
-gCGVbDf3n5KhOTnwqguELRykynKFt2LVksBia9+88lUtiRHpbyClo/KVWltJlaww
-PT0WEpwqVUcHmwrR3MTzJDEPvIplSgxdaDmFGYS1YKm9T/wQd+t/0DbXMmfJXBbd
-FVUnB6o7qJVU9N2Tbaj9NbCtwz5nTZG4A5kRXWHVjZsn5WzLuS/me3rDXjwlfB2p
-ipY=
------END CERTIFICATE-----
diff --git a/pulsar-broker/src/test/resources/certificate/server.csr b/pulsar-broker/src/test/resources/certificate/server.csr
deleted file mode 100644
index 8782222c5ab46..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/server.csr
+++ /dev/null
@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICnjCCAYYCAQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx
-ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJbG9j
-YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArNvSLs2b5Dhl
-JEv8Kis3f4dnrKowko8OEiy23odRgN1BL4dZPfnkYmMCRtcgjh5nLKeAPA7vffMW
-4HooaVlsO6Yy+VVZMt3Irrx8s9QnOmykabPxs1SQcBT9IMx5yvG1DPOlOdshk9Q3
-qQ/2NpVEctIoHVXADIxsHjMKE5omlXMGXsX2LhHn/o8935qZjsD/a26CAPa0K5k2
-y885GwMGV3Zfm0byHnyIyGsyM0zrPLSVq4RWkyVIAYYS4xSME3aUvp/hajrvCfdi
-vj7yoyHid5Jp2ckRyu7onZQJ9IPwyvnKNbF98zkTwlnoNtSvv71R04/dcwETWzvY
-VUuG6qrMPQIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAEPHySnpf3E/7tZsiDka
-rqdB/sU7fdqjyV0iy0cuKQkU8WYrsE7bHkqMYc8CiIDfWhIGW5Jnzups2O6eH0Sx
-2BS21ARFiNGC1UfY1HSV2zrTNh3RqQa3YsXzv9vvdQ/gjsqGDuGDIc1yAA+Ytdja
-3rhIzEVqBhiLzg+M2+gW1zs+Kqj0Zo0pLB2uqhdZJmjxBb2FCli50vCVEhqIS3RO
-KTE+AJfxThWIeahFyVaskaEGkS6NVr2JihV0elbKolH19k2UzRTVn7p3Ixh5ojuW
-gtU/90vOy/SDkSRmCWMqgkUKJ2oeImleHdrvwNyrzvrLWRAz6R5yGQJwji9kKpHD
-FK0=
------END CERTIFICATE REQUEST-----
diff --git a/pulsar-broker/src/test/resources/certificate/server.key b/pulsar-broker/src/test/resources/certificate/server.key
deleted file mode 100644
index 6da70f5aec3b5..0000000000000
--- a/pulsar-broker/src/test/resources/certificate/server.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCs29IuzZvkOGUk
-S/wqKzd/h2esqjCSjw4SLLbeh1GA3UEvh1k9+eRiYwJG1yCOHmcsp4A8Du998xbg
-eihpWWw7pjL5VVky3ciuvHyz1Cc6bKRps/GzVJBwFP0gzHnK8bUM86U52yGT1Dep
-D/Y2lURy0igdVcAMjGweMwoTmiaVcwZexfYuEef+jz3fmpmOwP9rboIA9rQrmTbL
-zzkbAwZXdl+bRvIefIjIazIzTOs8tJWrhFaTJUgBhhLjFIwTdpS+n+FqOu8J92K+
-PvKjIeJ3kmnZyRHK7uidlAn0g/DK+co1sX3zORPCWeg21K+/vVHTj91zARNbO9hV
-S4bqqsw9AgMBAAECggEAd/LuDeZFZ/+uR5qmuAhXMZqfWZSbsges5vW6S/6wkvB1
-vGp6heQzFAbKXKgJgjUcuULeXE6s58RYuppqEnin/1hcBOKxy/dUu9Q14H+2XPdo
-u6TPcvaaZ/xYjnr1hNtnHD6yB8zEpxVbLmjSHJxF7Dti9MA9TTfgCrC2LFYKsicD
-/5AQyHuwpHyTL3Iiwv4Qtks/SD2a3fu8lD0yTQwA/hY6/0ieXxXd9tZV5a6GSA0P
-nieol1byfuX7Q5fb8ggPd9u9K1mVZTBRKiE5w+uU4Ic2IkBmZX5ZuRS+vFplpLsY
-YpFPvzFmpNkpK2SdYjJ+V4tkJsFHmOaFRgW/0QB2DQKBgQDeQMSZBQlPUrgRdWHN
-OyvTcrSvXzg5DbaIj39tgdNZ6PYns/thD0n707KGRJOChIyYiiKxLxzLWdPUxqQO
-rNLUV9IkMVc/QZR8RUqGc2BxmPOxAprhzeOhLsyqP/sgtxRHAnLqmkXuHYoxvTZ6
-LFCRCZBpEJrutGxl3s/x+sfkuwKBgQDHGwnSmvArpL8ZY1dV4xKNkxifCBnNmqAl
-TKHPW3odN9nkMECEt1XUIioUUKXUsiAZNp5xa/v1DEyJ4f2T20QKcAGbS18b1M5W
-axIoH3IhyLo74tuo0fthgq5bzypfFOlIjo7F9mpEky/461RWmoNAAlp9+FkDi48C
-KwjAk39/ZwKBgQDXFJqs8sDFsOlMi+nvsHmDERhmNqG0JN8mXKgWk3KzKc09MuHs
-Vd1lBMNZSHfv8NIWtGdKTKty5yUmXm1ZfkoxECPevpkOMCq/8FZksrb8d+YswLae
-Gp9U1nNdtrkSOdo3tdj7y/wsqQ2ZgOB9bvEwyq6j3lvw8U2NcAiQxf44DQKBgBHb
-lPf0uZHQhutKA61KXoGgLdclrNrKAY8W3nRwqfUw6zQSN9cvcl1Cay/DQ/xdtY9N
-XMyjeMezwLGlOU8nnWSqQxqgmfkvDwqlM82xdFUfYcS5RiZQHxHR3L2TSSOaBoph
-buDGhyV7ZhQXV0slNJxrGZ6uxZ0RyVPSdEiBcjAFAoGBAJqZ6uCVHpv/FwZVggu7
-Xb9EIxZnLSmXwaXFpJoMZpRpKb8cSTTJbgSMv3Dq2LcNKYXdNBhgKgPSc/XipXt9
-ZdT36KWipV+PzW691kUiWHtA8/+E0LCi4Y7rlcBMz9PgDNXK4XMMZOVKxDqPcHSJ
-P6y01ku7T2X+abUiJ334Hg6G
------END PRIVATE KEY-----
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java
index 8d416125fd1b3..c401f3d0bea64 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java
@@ -328,9 +328,8 @@ public void testArgs() throws Exception {
         PulsarClientTool pulsarClientTool = new PulsarClientTool(new Properties());
         final String url = "pulsar+ssl://localhost:6651";
         final String authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls";
-        final String authParams = "tlsCertFile:pulsar-broker/src/test/resources/authentication/tls/client-cert.pem," +
-                "tlsKeyFile:pulsar-broker/src/test/resources/authentication/tls/client-key.pem";
-        final String tlsTrustCertsFilePath = "pulsar/pulsar-broker/src/test/resources/authentication/tls/cacert.pem";
+        final String authParams = String.format("tlsCertFile:%s,tlsKeyFile:%s", getTlsFileForClient("admin.cert"),
+                getTlsFileForClient("admin.key-pk8"));
         final String message = "test msg";
         final int numberOfMessages = 1;
         final String topicName = getTopicWithRandomSuffix("test-topic");
@@ -338,11 +337,11 @@ public void testArgs() throws Exception {
         String[] args = {"--url", url,
                 "--auth-plugin", authPlugin,
                 "--auth-params", authParams,
-                "--tlsTrustCertsFilePath", tlsTrustCertsFilePath,
+                "--tlsTrustCertsFilePath", CA_CERT_FILE_PATH,
                 "produce", "-m", message,
                 "-n", Integer.toString(numberOfMessages), topicName};
         pulsarClientTool.jcommander.parse(args);
-        assertEquals(pulsarClientTool.rootParams.getTlsTrustCertsFilePath(), tlsTrustCertsFilePath);
+        assertEquals(pulsarClientTool.rootParams.getTlsTrustCertsFilePath(), CA_CERT_FILE_PATH);
         assertEquals(pulsarClientTool.rootParams.getAuthParams(), authParams);
         assertEquals(pulsarClientTool.rootParams.getAuthPluginClassName(), authPlugin);
         assertEquals(pulsarClientTool.rootParams.getServiceURL(), url);
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java
index 0bc7c525384df..01c06fbf52f4e 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java
@@ -47,10 +47,6 @@
 import static org.testng.Assert.assertTrue;
 
 public class ProxyServiceTlsStarterTest extends MockedPulsarServiceBaseTest {
-
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
     private ProxyServiceStarter serviceStarter;
     private String serviceUrl;
     private int webPort;
@@ -63,14 +59,14 @@ protected void setup() throws Exception {
         serviceStarter.getConfig().setBrokerServiceURL(pulsar.getBrokerServiceUrl());
         serviceStarter.getConfig().setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls());
         serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress());
-        serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
+        serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH);
         serviceStarter.getConfig().setServicePort(Optional.empty());
         serviceStarter.getConfig().setServicePortTls(Optional.of(0));
         serviceStarter.getConfig().setWebServicePort(Optional.of(0));
         serviceStarter.getConfig().setTlsEnabledWithBroker(true);
         serviceStarter.getConfig().setWebSocketServiceEnabled(true);
-        serviceStarter.getConfig().setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        serviceStarter.getConfig().setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        serviceStarter.getConfig().setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        serviceStarter.getConfig().setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
         serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*");
         serviceStarter.start();
         serviceUrl = serviceStarter.getProxyService().getServiceUrlTls();
@@ -79,8 +75,8 @@ protected void setup() throws Exception {
 
     protected void doInitConf() throws Exception {
         super.doInitConf();
-        this.conf.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        this.conf.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        this.conf.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        this.conf.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
     }
 
     @Override
@@ -94,7 +90,7 @@ protected void cleanup() throws Exception {
     public void testProducer() throws Exception {
         @Cleanup
         PulsarClient client = PulsarClient.builder().serviceUrl(serviceUrl)
-                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
+                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
                 .build();
 
         @Cleanup
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java
index e1cf62aafa8a3..64b0cd6b1a610 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java
@@ -43,10 +43,6 @@
 
 public class ProxyTlsTest extends MockedPulsarServiceBaseTest {
 
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
-
     private ProxyService proxyService;
     private ProxyConfiguration proxyConfig = new ProxyConfiguration();
 
@@ -61,8 +57,8 @@ protected void setup() throws Exception {
         proxyConfig.setWebServicePort(Optional.of(0));
         proxyConfig.setWebServicePortTls(Optional.of(0));
         proxyConfig.setTlsEnabledWithBroker(false);
-        proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
         proxyConfig.setMetadataStoreUrl(DUMMY_VALUE);
         proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE);
 
@@ -87,7 +83,7 @@ public void testProducer() throws Exception {
         @Cleanup
         PulsarClient client = PulsarClient.builder()
                 .serviceUrl(proxyService.getServiceUrlTls())
-                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).build();
+                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
         Producer producer = client.newProducer(Schema.BYTES).topic("persistent://sample/test/local/topic").create();
 
         for (int i = 0; i < 10; i++) {
@@ -100,7 +96,7 @@ public void testPartitions() throws Exception {
         @Cleanup
         PulsarClient client = PulsarClient.builder()
                 .serviceUrl(proxyService.getServiceUrlTls())
-                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).build();
+                .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(CA_CERT_FILE_PATH).build();
         TenantInfoImpl tenantInfo = createDefaultTenantInfo();
         admin.tenants().createTenant("sample", tenantInfo);
         admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
index d5b70dfa03756..f77c0eeb2d41c 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
@@ -35,10 +35,6 @@
 
 public class ProxyTlsTestWithAuth extends MockedPulsarServiceBaseTest {
 
-    private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
-    private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem";
-    private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem";
-
     private ProxyService proxyService;
     private ProxyConfiguration proxyConfig = new ProxyConfiguration();
 
@@ -63,8 +59,8 @@ protected void setup() throws Exception {
         proxyConfig.setWebServicePort(Optional.of(0));
         proxyConfig.setWebServicePortTls(Optional.of(0));
         proxyConfig.setTlsEnabledWithBroker(true);
-        proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH);
-        proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH);
+        proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH);
+        proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH);
         proxyConfig.setMetadataStoreUrl(DUMMY_VALUE);
         proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE);
         proxyConfig.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2");

From b88c48ac17a3a034abc0b975ed3ddd06e51c1c47 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sat, 20 May 2023 00:28:35 +0800
Subject: [PATCH 299/404] [improve][broker] Copy BrokerEntryMetadata when
 rebatchMessage (#20337)

---
 .../pulsar/client/impl/RawBatchConverter.java | 21 +++++++++-
 .../pulsar/client/impl/RawReaderTest.java     | 40 +++++++++++++++++++
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
index 4d718f71a2e23..4809ce1a04807 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
@@ -19,7 +19,9 @@
 package org.apache.pulsar.client.impl;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.pulsar.common.protocol.Commands.magicBrokerEntryMetadata;
 import io.netty.buffer.ByteBuf;
+import io.netty.buffer.CompositeByteBuf;
 import io.netty.buffer.Unpooled;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -92,6 +94,14 @@ public static Optional rebatchMessage(RawMessage msg,
         checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
 
         ByteBuf payload = msg.getHeadersAndPayload();
+        int readerIndex = payload.readerIndex();
+        ByteBuf brokerMeta = null;
+        if (payload.getShort(readerIndex) == magicBrokerEntryMetadata) {
+            payload.skipBytes(Short.BYTES);
+            int brokerEntryMetadataSize = payload.readInt();
+            payload.readerIndex(readerIndex);
+            brokerMeta = payload.readRetainedSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES);
+        }
         MessageMetadata metadata = Commands.parseMessageMetadata(payload);
         ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity());
 
@@ -139,8 +149,15 @@ public static Optional rebatchMessage(RawMessage msg,
 
                 ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c,
                                                                                   metadata, compressedPayload);
-                Optional result = Optional.of(new RawMessageImpl(msg.getMessageIdData(),
-                                                                             metadataAndPayload));
+
+                if (brokerMeta != null) {
+                    CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeDirectBuffer();
+                    compositeByteBuf.addComponents(true, brokerMeta, metadataAndPayload);
+                    metadataAndPayload = compositeByteBuf;
+                }
+
+                Optional result =
+                        Optional.of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload));
                 metadataAndPayload.release();
                 compressedPayload.release();
                 return result;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
index fb09cd9951885..a201ef104e7b3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
@@ -39,6 +39,7 @@
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.RawMessage;
 import org.apache.pulsar.client.api.RawReader;
+import org.apache.pulsar.common.api.proto.BrokerEntryMetadata;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
@@ -57,6 +58,11 @@ public class RawReaderTest extends MockedPulsarServiceBaseTest {
     @BeforeMethod
     @Override
     public void setup() throws Exception {
+        conf.setBrokerEntryMetadataInterceptors(org.assertj.core.util.Sets.newTreeSet(
+                "org.apache.pulsar.common.intercept.AppendBrokerTimestampMetadataInterceptor",
+                "org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor"
+        ));
+        conf.setExposingBrokerEntryMetadataToClientEnabled(true);
         super.internalSetup();
 
         admin.clusters().createCluster("test",
@@ -384,6 +390,40 @@ public void testBatchingRebatch() throws Exception {
         }
     }
 
+    @Test
+    public void testBatchingRebatchWithBrokerEntryMetadata() throws Exception {
+        String topic = "persistent://my-property/my-ns/my-raw-topic";
+
+        try (Producer producer = pulsarClient.newProducer().topic(topic)
+                .maxPendingMessages(3)
+                .enableBatching(true)
+                .batchingMaxMessages(3)
+                .batchingMaxPublishDelay(1, TimeUnit.HOURS)
+                .messageRoutingMode(MessageRoutingMode.SinglePartition)
+                .create()) {
+            producer.newMessage().key("key1").value("my-content-1".getBytes()).sendAsync();
+            producer.newMessage().key("key2").value("my-content-2".getBytes()).sendAsync();
+            producer.newMessage().key("key3").value("my-content-3".getBytes()).send();
+        }
+
+        RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
+        try (RawMessage m1 = reader.readNextAsync().get()) {
+            RawMessage m2 = RawBatchConverter.rebatchMessage(m1, (key, id) -> key.equals("key2")).get();
+            BrokerEntryMetadata brokerEntryMetadata =
+                    Commands.parseBrokerEntryMetadataIfExist(m2.getHeadersAndPayload());
+            Assert.assertNotNull(brokerEntryMetadata);
+            Assert.assertEquals(brokerEntryMetadata.getIndex(), 2);
+            Assert.assertTrue(brokerEntryMetadata.getBrokerTimestamp() < System.currentTimeMillis());
+            List> idsAndKeys = RawBatchConverter.extractIdsAndKeysAndSize(m2);
+            Assert.assertEquals(idsAndKeys.size(), 1);
+            Assert.assertEquals(idsAndKeys.get(0).getMiddle(), "key2");
+            m2.close();
+            Assert.assertEquals(m1.getHeadersAndPayload().refCnt(), 1);
+        } finally {
+            reader.closeAsync().get();
+        }
+    }
+
     @Test
     public void testAcknowledgeWithProperties() throws Exception {
         int numKeys = 10;

From 1e664b7f550ffa28d3c810f3b7d6d625d5905eb3 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Sat, 20 May 2023 03:38:24 +0800
Subject: [PATCH 300/404] [fix] [broker] In Key_Shared mode: remove unnecessary
 mechanisms of message skip to avoid unnecessary consumption stuck (#20335)

### Motivation
- https://github.com/apache/pulsar/pull/7105 provide a mechanism to avoid a stuck consumer affecting the consumption of other consumers:
  - if all consumers can not accept more messages, stop delivering messages to the client.
  - if one consumer can not accept more messages, just read new messages and deliver them to other consumers.
- https://github.com/apache/pulsar/pull/7553 provide a mechanism to fix the issue of lost order of consumption: If the consumer cannot accept any more messages, skip the consumer for the next round of message delivery because there may be messages with the same key in the replay queue.
- https://github.com/apache/pulsar/pull/10762 provide a mechanism to fix the issue of lost order of consumption: If there have any messages with the same key in the replay queue, do not deliver the new messages to this consumer.

https://github.com/apache/pulsar/pull/10762 and https://github.com/apache/pulsar/pull/7553 do the same thing and https://github.com/apache/pulsar/pull/10762 is better than https://github.com/apache/pulsar/pull/7553 , so https://github.com/apache/pulsar/pull/7553 is unnecessary.

### Modifications
remove the mechanism provided by https://github.com/apache/pulsar/pull/7553 to avoid unnecessary consumption stuck.
---
 ...tStickyKeyDispatcherMultipleConsumers.java | 24 +------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index 1a8c6e180a2a2..8f05530f58bfa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -71,17 +71,12 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
      */
     private final LinkedHashMap recentlyJoinedConsumers;
 
-    private final Set stuckConsumers;
-    private final Set nextStuckConsumers;
-
     PersistentStickyKeyDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor,
             Subscription subscription, ServiceConfiguration conf, KeySharedMeta ksm) {
         super(topic, cursor, subscription, ksm.isAllowOutOfOrderDelivery());
 
         this.allowOutOfOrderDelivery = ksm.isAllowOutOfOrderDelivery();
         this.recentlyJoinedConsumers = allowOutOfOrderDelivery ? null : new LinkedHashMap<>();
-        this.stuckConsumers = new HashSet<>();
-        this.nextStuckConsumers = new HashSet<>();
         this.keySharedMode = ksm.getKeySharedMode();
         switch (this.keySharedMode) {
         case AUTO_SPLIT:
@@ -226,8 +221,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
             }
         }
 
-        nextStuckConsumers.clear();
-
         final Map> groupedEntries = localGroupedEntries.get();
         groupedEntries.clear();
         final Map> consumerStickyKeyHashesMap = new HashMap<>();
@@ -318,14 +311,11 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
         // acquire message-dispatch permits for already delivered messages
         acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent);
 
-        stuckConsumers.clear();
-
         if (totalMessagesSent == 0 && (recentlyJoinedConsumers == null || recentlyJoinedConsumers.isEmpty())) {
             // This means, that all the messages we've just read cannot be dispatched right now.
             // This condition can only happen when:
             //  1. We have consumers ready to accept messages (otherwise the would not haven been triggered)
             //  2. All keys in the current set of messages are routing to consumers that are currently busy
-            //     and stuck is not caused by stuckConsumers
             //
             // The solution here is to move on and read next batch of messages which might hopefully contain
             // also keys meant for other consumers.
@@ -334,10 +324,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
             // ahead in the stream while the new consumers are not ready to accept the new messages,
             // therefore would be most likely only increase the distance between read-position and mark-delete
             // position.
-            if (!nextStuckConsumers.isEmpty()) {
-                isDispatcherStuckOnReplays = true;
-                stuckConsumers.addAll(nextStuckConsumers);
-            }
+            isDispatcherStuckOnReplays = true;
             return true;
         }  else if (currentThreadKeyNumber == 0) {
             return true;
@@ -348,8 +335,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis
     private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List entries, int maxMessages,
             ReadType readType, Set stickyKeyHashes) {
         if (maxMessages == 0) {
-            // the consumer was stuck
-            nextStuckConsumers.add(consumer);
             return 0;
         }
         if (readType == ReadType.Normal && stickyKeyHashes != null
@@ -366,13 +351,6 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en
         // At this point, all the old messages were already consumed and this consumer
         // is now ready to receive any message
         if (maxReadPosition == null) {
-            // stop to dispatch by stuckConsumers
-            if (stuckConsumers.contains(consumer)) {
-                if (log.isDebugEnabled()) {
-                    log.debug("[{}] stop to dispatch by stuckConsumers, consumer: {}", name, consumer);
-                }
-                return 0;
-            }
             // The consumer has not recently joined, so we can send all messages
             return maxMessages;
         }

From eae4bd069e3af60c2a8815453dc057c768dce4a9 Mon Sep 17 00:00:00 2001
From: Paul Gier 
Date: Sat, 20 May 2023 11:34:07 -0500
Subject: [PATCH 301/404] [fix][broker] update rest endpoint method names to
 avoid swagger conflict (#20359)

Signed-off-by: Paul Gier 
---
 .../pulsar/broker/admin/v2/Namespaces.java    |  6 ++--
 src/gen-swagger.sh                            | 33 -------------------
 2 files changed, 3 insertions(+), 36 deletions(-)
 delete mode 100755 src/gen-swagger.sh

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
index 55766c173f71f..b4f1194f92f0a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -2519,7 +2519,7 @@ public void getMaxTopicsPerNamespace(@Suspended final AsyncResponse asyncRespons
     @ApiOperation(value = "Set maxTopicsPerNamespace config on a namespace.")
     @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
             @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), })
-    public void setInactiveTopicPolicies(@PathParam("tenant") String tenant,
+    public void setMaxTopicsPerNamespace(@PathParam("tenant") String tenant,
                                          @PathParam("namespace") String namespace,
                                          @ApiParam(value = "Number of maximum topics for specific namespace",
                                                  required = true) int maxTopicsPerNamespace) {
@@ -2529,10 +2529,10 @@ public void setInactiveTopicPolicies(@PathParam("tenant") String tenant,
 
     @DELETE
     @Path("/{tenant}/{namespace}/maxTopicsPerNamespace")
-    @ApiOperation(value = "Set maxTopicsPerNamespace config on a namespace.")
+    @ApiOperation(value = "Remove maxTopicsPerNamespace config on a namespace.")
     @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
             @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), })
-    public void setInactiveTopicPolicies(@PathParam("tenant") String tenant,
+    public void removeMaxTopicsPerNamespace(@PathParam("tenant") String tenant,
                                          @PathParam("namespace") String namespace) {
         validateNamespaceName(tenant, namespace);
         internalRemoveMaxTopicsPerNamespace();
diff --git a/src/gen-swagger.sh b/src/gen-swagger.sh
deleted file mode 100755
index 6402dca261408..0000000000000
--- a/src/gen-swagger.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-PULSAR_PATH=$(git rev-parse --show-toplevel)
-
-cd $PULSAR_PATH
-
-echo "Generating swagger json file for master ..."
-mvn -am -pl pulsar-broker install -DskipTests -Pswagger
-echo "Swagger json file is generated for master."
-
-mkdir -p site2/website/static/swagger/master/
-
-cp pulsar-broker/target/docs/swagger*.json site2/website/static/swagger/master/
-echo "Copied swagger json file for master."
-

From aa7decc5b75894e864f3c5962c5cffad255abf25 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Sat, 20 May 2023 09:34:27 -0700
Subject: [PATCH 302/404] [fix][broker]
 managedLedger.getConfig().getProperties().putAll(properties) NPE (#20361)

---
 .../apache/pulsar/broker/admin/impl/PersistentTopicsBase.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index ed7cd70c6411c..e0f168eb8d842 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -699,7 +699,11 @@ private CompletableFuture internalUpdateNonPartitionedTopicProperties(Map<
 
                     @Override
                     public void updatePropertiesComplete(Map properties, Object ctx) {
+                        if (managedLedger.getConfig().getProperties() == null) {
+                            managedLedger.getConfig().setProperties(new HashMap<>());
+                        }
                         managedLedger.getConfig().getProperties().putAll(properties);
+
                         future.complete(null);
                     }
 

From a9f2f28891b4c0f16c31e0b5ec8667e396479e90 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Sun, 21 May 2023 17:50:44 -0700
Subject: [PATCH 303/404] [improve][broker] Gracefully shut down load balancer
 extension (#20315)

### Motivation

Load balancer extension needs to shut down gracefully, especially when shutting down the leader broker. When the leader broker closes the leader election service too late, service unit lookups to the leader broker could fail during the shutdown. This could delay client reconnection time.

Lookup failure logs
```
(shutdown starts)
pulsar-broker-8 pulsar-broker 2023-04-22T00:19:52,630+0000 [pulsar-service-shutdown] INFO  org.apache.pulsar.broker.PulsarService - Closing PulsarService
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,690+0000 [pulsar-io-4-5] INFO  org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [pulsar-18-9] Closed connection [id: 0x907e74b0, L:/10.0.13.6:35838 ! R:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local/10.0.18.18:6650] -- Will try again in 0.1 s
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,691+0000 [pulsar-io-4-5] INFO  org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [reader-30e6b40dd9] Closed connection [id: 0x907e74b0, L:/10.0.13.6:35838 ! R:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local/10.0.18.18:6650] -- Will try again in 0.1 s

(znode is gone)
pulsar-broker-8 pulsar-broker 2023-04-22T00:19:52,652+0000 [pulsar-load-manager-1-1] INFO  org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl - BrokerRegistry detected the broker:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 registry has been deleted.

(lookup failure)
org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [reader-30e6b40dd9] Could not get connection to broker: org.apache.pulsar.client.api.PulsarClientException$ConnectException: Disconnected from server at pulsar-broker-3.pulsar-broker.pulsar.svc.cluster.local/10.0.13.6:6650 -- Will try again in 0.193 s
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,827+0000 [main-EventThread] INFO  org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup null for topic persistent://pulsar/system/loadbalancer-service-unit-state with error Failed to look up a broker registry:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 for bundle:pulsar/system/0x40000000_0x50000000
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,827+0000 [main-EventThread] INFO  org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup null for topic persistent://pulsar/system/loadbalancer-service-unit-state with error Failed to look up a broker registry:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 for bundle:pulsar/system/0x40000000_0x50000000

(leader election service has been closed)
pulsar-broker-3 pulsar-broker 2023-04-22T00:19:55,569+0000 [pulsar-load-manager-1-1] INFO  org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl - This broker:pulsar-broker-3.pulsar-broker.pulsar.svc.cluster.local:8080 plays the leader now.

```

### Modifications
During the shutdown flow, when calling loadbalancer.disableBroker(), the load balancer extension gracefully gives up the owned bundles to other brokers. After that, it closes the leader election service and removes its register in the metadata store.
---
 .../apache/pulsar/broker/PulsarService.java   |  22 +++-
 .../extensions/ExtensibleLoadManagerImpl.java |  17 +++
 .../ExtensibleLoadManagerWrapper.java         |   2 +-
 .../channel/ServiceUnitStateChannel.java      |   5 +
 .../channel/ServiceUnitStateChannelImpl.java  | 112 +++++++++++++++---
 .../ExtensibleLoadManagerImplTest.java        |  45 +++++++
 .../channel/ServiceUnitStateChannelTest.java  |  65 ++++++----
 7 files changed, 222 insertions(+), 46 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index f833674ceeb0d..62d4634fa2d62 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -382,6 +382,17 @@ public void closeMetadataServiceSession() throws Exception {
         localMetadataStore.close();
     }
 
+    private void closeLeaderElectionService() throws Exception {
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService().close();
+        } else {
+            if (this.leaderElectionService != null) {
+                this.leaderElectionService.close();
+                this.leaderElectionService = null;
+            }
+        }
+    }
+
     @Override
     public void close() throws PulsarServerException {
         try {
@@ -502,10 +513,7 @@ public CompletableFuture closeAsync() {
                 this.bkClientFactory = null;
             }
 
-            if (this.leaderElectionService != null) {
-                this.leaderElectionService.close();
-                this.leaderElectionService = null;
-            }
+            closeLeaderElectionService();
 
             if (adminClient != null) {
                 adminClient.close();
@@ -1316,7 +1324,11 @@ public boolean isRunning() {
      * @return a reference of the current LeaderElectionService instance.
      */
     public LeaderElectionService getLeaderElectionService() {
-        return this.leaderElectionService;
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            return ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService();
+        } else {
+            return this.leaderElectionService;
+        }
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 4ebf537f7a8a8..cbaed8ee8f94f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -26,6 +26,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -380,12 +381,22 @@ public CompletableFuture> assign(Optional> selectAsync(ServiceUnitId bundle) {
+        return selectAsync(bundle, Collections.emptySet());
+    }
+
+    public CompletableFuture> selectAsync(ServiceUnitId bundle,
+                                                           Set excludeBrokerSet) {
         BrokerRegistry brokerRegistry = getBrokerRegistry();
         return brokerRegistry.getAvailableBrokerLookupDataAsync()
                 .thenCompose(availableBrokers -> {
                     LoadManagerContext context = this.getContext();
 
                     Map availableBrokerCandidates = new HashMap<>(availableBrokers);
+                    if (!excludeBrokerSet.isEmpty()) {
+                        for (String exclude : excludeBrokerSet) {
+                            availableBrokerCandidates.remove(exclude);
+                        }
+                    }
 
                     // Filter out brokers that do not meet the rules.
                     List filterPipeline = getBrokerFilterPipeline();
@@ -685,4 +696,10 @@ private void monitor() {
             log.error("Failed to get the channel ownership.", e);
         }
     }
+
+    public void disableBroker() throws Exception {
+        serviceUnitStateChannel.cleanOwnerships();
+        leaderElectionService.close();
+        brokerRegistry.unregister();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java
index 1eabbe620e213..18e949537dedb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java
@@ -74,7 +74,7 @@ public CompletableFuture checkOwnershipAsync(Optional to
 
     @Override
     public void disableBroker() throws Exception {
-        this.loadManager.getBrokerRegistry().unregister();
+        this.loadManager.disableBroker();
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index 4782a31fe0c56..6e75fe91a914f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -206,4 +206,9 @@ public interface ServiceUnitStateChannel extends Closeable {
      * Cancels the ownership monitor.
      */
     void cancelOwnershipMonitor();
+
+    /**
+     * Cleans the service unit ownerships from the current broker's channel.
+     */
+    void cleanOwnerships();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index b4c4e7fd5d42a..6246c26e57bcc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -91,7 +91,6 @@
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
 import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceBundles;
-import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.stats.Metrics;
@@ -110,6 +109,10 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
 
     public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD;
     private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec
+
+    private static final int OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS = 5000;
+    private static final int OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS = 100;
+    private static final int OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS = 3000;
     public static final long VERSION_ID_INIT = 1; // initial versionId
     private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60;
     public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins
@@ -257,6 +260,11 @@ public void cancelOwnershipMonitor() {
         }
     }
 
+    @Override
+    public void cleanOwnerships() {
+        doCleanup(lookupServiceAddress);
+    }
+
     public synchronized void start() throws PulsarServerException {
         if (!validateChannelState(LeaderElectionServiceStarted, false)) {
             throw new IllegalStateException("Invalid channel state:" + channelState.name());
@@ -284,7 +292,7 @@ public synchronized void start() throws PulsarServerException {
                 }
             }
             PulsarClusterMetadataSetup.createNamespaceIfAbsent
-                    (pulsar.getPulsarResources(), NamespaceName.SYSTEM_NAMESPACE, config.getClusterName());
+                    (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, config.getClusterName());
 
             producer = pulsar.getClient().newProducer(schema)
                     .enableBatching(true)
@@ -694,6 +702,8 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) {
         if (isTargetBroker(data.dstBroker())) {
             log(null, serviceUnit, data, null);
             lastOwnEventHandledAt = System.currentTimeMillis();
+        } else if (data.force() && isTargetBroker(data.sourceBroker())) {
+            closeServiceUnit(serviceUnit);
         }
     }
 
@@ -1108,21 +1118,23 @@ private void scheduleCleanup(String broker, long delayInSecs) {
 
 
     private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData,
-                                                                    String selectedBroker) {
+                                                                    String selectedBroker,
+                                                                    String inactiveBroker) {
         if (orphanData.state() == Splitting) {
             return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker,
                     Map.copyOf(orphanData.splitServiceUnitToDestBroker()),
                     true, getNextVersionId(orphanData));
         } else {
-            return new ServiceUnitStateData(Owned, selectedBroker, true, getNextVersionId(orphanData));
+            return new ServiceUnitStateData(Owned, selectedBroker, inactiveBroker,
+                    true, getNextVersionId(orphanData));
         }
     }
 
-    private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) {
-
-        Optional selectedBroker = selectBroker(serviceUnit);
+    private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, String inactiveBroker) {
+        Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker);
         if (selectedBroker.isPresent()) {
-            var override = getOverrideInactiveBrokerStateData(orphanData, selectedBroker.get());
+            var override = getOverrideInactiveBrokerStateData(
+                    orphanData, selectedBroker.get(), inactiveBroker);
             log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}",
                     serviceUnit, orphanData, override);
             publishOverrideEventAsync(serviceUnit, orphanData, override)
@@ -1140,26 +1152,69 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa
         }
     }
 
+    private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) {
+        long started = System.currentTimeMillis();
+        while (System.currentTimeMillis() - started < maxWaitTimeInMillis) {
+            boolean cleaned = true;
+            for (var etr : tableview.entrySet()) {
+                var serviceUnit = etr.getKey();
+                var data = etr.getValue();
+
+                if (excludeSystemTopics && serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                    continue;
+                }
+
+                if (data.state() == Owned && broker.equals(data.dstBroker())) {
+                    cleaned = false;
+                    break;
+                }
+            }
+            if (cleaned) {
+                try {
+                    MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS);
+                } catch (InterruptedException e) {
+                    log.warn("Interrupted while gracefully waiting for the cleanup convergence.");
+                }
+                break;
+            } else {
+                try {
+                    MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS);
+                } catch (InterruptedException e) {
+                    log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}",
+                            lookupServiceAddress);
+                }
+            }
+        }
+    }
 
-    private void doCleanup(String broker)  {
+    private synchronized void doCleanup(String broker)  {
         long startTime = System.nanoTime();
         log.info("Started ownership cleanup for the inactive broker:{}", broker);
         int orphanServiceUnitCleanupCnt = 0;
         long totalCleanupErrorCntStart = totalCleanupErrorCnt.get();
 
+        Map orphanSystemServiceUnits = new HashMap<>();
         for (var etr : tableview.entrySet()) {
             var stateData = etr.getValue();
             var serviceUnit = etr.getKey();
             var state = state(stateData);
             if (StringUtils.equals(broker, stateData.dstBroker())) {
                 if (isActiveState(state)) {
-                    overrideOwnership(serviceUnit, stateData);
+                    if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                        orphanSystemServiceUnits.put(serviceUnit, stateData);
+                    } else {
+                        overrideOwnership(serviceUnit, stateData, broker);
+                    }
                     orphanServiceUnitCleanupCnt++;
                 }
 
             } else if (StringUtils.equals(broker, stateData.sourceBroker())) {
                 if (isInFlightState(state)) {
-                    overrideOwnership(serviceUnit, stateData);
+                    if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) {
+                        orphanSystemServiceUnits.put(serviceUnit, stateData);
+                    } else {
+                        overrideOwnership(serviceUnit, stateData, broker);
+                    }
                     orphanServiceUnitCleanupCnt++;
                 }
             }
@@ -1168,14 +1223,33 @@ private void doCleanup(String broker)  {
         try {
             producer.flush();
         } catch (PulsarClientException e) {
-            log.error("Failed to flush the in-flight messages.", e);
+            log.error("Failed to flush the in-flight non-system bundle override messages.", e);
         }
 
+
         if (orphanServiceUnitCleanupCnt > 0) {
+            // System bundles can contain this channel's system topic and other important system topics.
+            // Cleaning such system bundles(closing the system topics) together with the non-system bundles
+            // can cause the cluster to be temporarily unstable.
+            // Hence, we clean the non-system bundles first and gracefully wait for them.
+            // After that, we clean the system bundles, if any.
+            waitForCleanups(broker, true, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS);
             this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt;
             this.totalInactiveBrokerCleanupCnt++;
         }
 
+        // clean system bundles in the end
+        for (var orphanSystemServiceUnit : orphanSystemServiceUnits.entrySet()) {
+            log.info("Overriding orphan system service unit:{}", orphanSystemServiceUnit.getKey());
+            overrideOwnership(orphanSystemServiceUnit.getKey(), orphanSystemServiceUnit.getValue(), broker);
+        }
+
+        try {
+            producer.flush();
+        } catch (PulsarClientException e) {
+            log.error("Failed to flush the in-flight system bundle override messages.", e);
+        }
+
         double cleanupTime = TimeUnit.NANOSECONDS
                 .toMillis((System.nanoTime() - startTime));
 
@@ -1194,9 +1268,9 @@ private void doCleanup(String broker)  {
 
     }
 
-    private Optional selectBroker(String serviceUnit) {
+    private Optional selectBroker(String serviceUnit, String inactiveBroker) {
         try {
-            return loadManager.selectAsync(getNamespaceBundle(serviceUnit))
+            return loadManager.selectAsync(getNamespaceBundle(serviceUnit), Set.of(inactiveBroker))
                     .get(inFlightStateWaitingTimeInMillis, MILLISECONDS);
         } catch (Throwable e) {
             log.error("Failed to select a broker for serviceUnit:{}", serviceUnit);
@@ -1204,8 +1278,10 @@ private Optional selectBroker(String serviceUnit) {
         return Optional.empty();
     }
 
-    private Optional getRollForwardStateData(String serviceUnit, long nextVersionId) {
-        Optional selectedBroker = selectBroker(serviceUnit);
+    private Optional getRollForwardStateData(String serviceUnit,
+                                                                   String inactiveBroker,
+                                                                   long nextVersionId) {
+        Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker);
         if (selectedBroker.isEmpty()) {
             return Optional.empty();
         }
@@ -1220,7 +1296,7 @@ private Optional getOverrideInFlightStateData(
         var state = orphanData.state();
         switch (state) {
             case Assigning: {
-                return getRollForwardStateData(serviceUnit, nextVersionId);
+                return getRollForwardStateData(serviceUnit, orphanData.dstBroker(), nextVersionId);
             }
             case Splitting: {
                 return Optional.of(new ServiceUnitStateData(Splitting,
@@ -1233,7 +1309,7 @@ private Optional getOverrideInFlightStateData(
                     // rollback to the src
                     return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId));
                 } else {
-                    return getRollForwardStateData(serviceUnit, nextVersionId);
+                    return getRollForwardStateData(serviceUnit, orphanData.sourceBroker(), nextVersionId);
                 }
             }
             default: {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index b71eeb4745b87..b131aff68d909 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -880,6 +880,51 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
         assertEquals(actual, expected);
     }
 
+    @Test
+    public void testDisableBroker() throws Exception {
+        // Test rollback to modular load manager.
+        ServiceConfiguration defaultConf = getDefaultConf();
+        defaultConf.setAllowAutoTopicCreation(true);
+        defaultConf.setForceDeleteNamespaceAllowed(true);
+        defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+        defaultConf.setLoadBalancerSheddingEnabled(false);
+        try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) {
+            var pulsar3 = additionalPulsarTestContext.getPulsarService();
+            ExtensibleLoadManagerImpl ternaryLoadManager = spy((ExtensibleLoadManagerImpl)
+                    FieldUtils.readField(pulsar3.getLoadManager().get(), "loadManager", true));
+            String topic = "persistent://public/default/test";
+
+            String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic);
+            TopicName topicName = TopicName.get("test");
+            NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
+            if (!pulsar3.getBrokerServiceUrl().equals(lookupResult1)) {
+                admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(),
+                        pulsar3.getLookupServiceAddress());
+                lookupResult1 = pulsar2.getAdminClient().lookups().lookupTopic(topic);
+            }
+            String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic);
+            String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic);
+
+            assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl());
+            assertEquals(lookupResult1, lookupResult2);
+            assertEquals(lookupResult1, lookupResult3);
+
+
+            assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            assertTrue(ternaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+
+            ternaryLoadManager.disableBroker();
+
+            assertFalse(ternaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            if (primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()) {
+                assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            } else {
+                assertTrue(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+            }
+        }
+    }
+
     private static abstract class MockBrokerFilter implements BrokerFilter {
 
         @Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 77c80187a63e1..cb26c460f0a03 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -515,7 +515,7 @@ public void transferTestWhenDestBrokerFails()
 
         // recovered, check the monitor update state : Assigned -> Owned
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         FieldUtils.writeDeclaredField(channel2, "producer", producer, true);
         FieldUtils.writeDeclaredField(channel1,
                 "inFlightStateWaitingTimeInMillis", 1 , true);
@@ -713,8 +713,8 @@ public void handleBrokerDeletionEventTest()
 
         var cleanupJobs1 = getCleanupJobs(channel1);
         var cleanupJobs2 = getCleanupJobs(channel2);
-        var leaderCleanupJobs = spy(cleanupJobs1);
-        var followerCleanupJobs = spy(cleanupJobs2);
+        var leaderCleanupJobsTmp = spy(cleanupJobs1);
+        var followerCleanupJobsTmp = spy(cleanupJobs2);
         var leaderChannel = channel1;
         var followerChannel = channel2;
         String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get();
@@ -723,10 +723,12 @@ public void handleBrokerDeletionEventTest()
         if (leader.equals(lookupServiceAddress2)) {
             leaderChannel = channel2;
             followerChannel = channel1;
-            var tmp = followerCleanupJobs;
-            followerCleanupJobs = leaderCleanupJobs;
-            leaderCleanupJobs = tmp;
+            var tmp = followerCleanupJobsTmp;
+            followerCleanupJobsTmp = leaderCleanupJobsTmp;
+            leaderCleanupJobsTmp = tmp;
         }
+        final var leaderCleanupJobs = leaderCleanupJobsTmp;
+        final var followerCleanupJobs = followerCleanupJobsTmp;
         FieldUtils.writeDeclaredField(leaderChannel, "cleanupJobs", leaderCleanupJobs,
                 true);
         FieldUtils.writeDeclaredField(followerChannel, "cleanupJobs", followerCleanupJobs,
@@ -735,7 +737,7 @@ public void handleBrokerDeletionEventTest()
         var owner1 = channel1.getOwnerAsync(bundle1);
         var owner2 = channel2.getOwnerAsync(bundle2);
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         assertTrue(owner1.get().isEmpty());
         assertTrue(owner2.get().isEmpty());
 
@@ -769,9 +771,11 @@ public void handleBrokerDeletionEventTest()
         verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
 
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
 
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -797,8 +801,12 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(2)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(1, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(1, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -814,8 +822,12 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(2)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -833,8 +845,11 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(1, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(1, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 1,
                 0,
@@ -852,8 +867,11 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 2,
                 0,
@@ -878,8 +896,11 @@ public void handleBrokerDeletionEventTest()
 
         verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any());
         verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any());
-        assertEquals(0, leaderCleanupJobs.size());
-        assertEquals(0, followerCleanupJobs.size());
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertEquals(0, leaderCleanupJobs.size());
+            assertEquals(0, followerCleanupJobs.size());
+        });
+
         validateMonitorCounters(leaderChannel,
                 2,
                 0,
@@ -1101,7 +1122,7 @@ public void assignTestWhenDestBrokerProducerFails()
         FieldUtils.writeDeclaredField(channel2,
                 "inFlightStateWaitingTimeInMillis", 3 * 1000, true);
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         channel1.publishAssignEventAsync(bundle, lookupServiceAddress2);
         // channel1 is broken. the assign won't be complete.
         waitUntilState(channel1, bundle);
@@ -1440,7 +1461,7 @@ public void testOverrideInactiveBrokerStateData()
 
         // test stable metadata state
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         leaderChannel.handleMetadataSessionEvent(SessionReestablished);
         followerChannel.handleMetadataSessionEvent(SessionReestablished);
         FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp",
@@ -1505,7 +1526,7 @@ public void testOverrideOrphanStateData()
 
         // test stable metadata state
         doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2)))
-                .when(loadManager).selectAsync(any());
+                .when(loadManager).selectAsync(any(), any());
         FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis",
                 -1, true);
         FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis",

From b7f0004313ea4565717cc6d3c0b99aee5c079c6c Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Mon, 22 May 2023 08:51:10 +0800
Subject: [PATCH 304/404] [fix][broker] Fix broker load manager class filter
 NPE (#20350)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
When upgrading the pulsar version and changing the pulsar load manager to `ExtensibleLoadManagerImpl` it might cause NPE. The root cause is the old version of pulsar does not contain the `loadManagerClassName` field.
```
2023-05-18T05:42:50,557+0000 [pulsar-io-4-1] INFO  org.apache.pulsar.broker.service.ServerCnx - [/127.0.0.6:51345] connected with role=[pulsarinstance-v3-0-n@test.dev](mailto:pulsarinstance-v3-0-n@test.dev) using authMethod=token, clientVersion=Pulsar Go 0.9.0, clientProtocolVersion=18, proxyVersion=null
2023-05-18T05:42:50,558+0000 [pulsar-io-4-1] WARN  org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup [pulsarinstance-v3-0-n@test.dev](mailto:pulsarinstance-v3-0-n@test.dev) for topic persistent://xxx with error java.lang.NullPointerException: Cannot invoke “String.equals(Object)” because the return value of “org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData.getLoadManagerClassName()” is null
java.util.concurrent.CompletionException: java.lang.NullPointerException: Cannot invoke “String.equals(Object)” because the return value of “org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData.getLoadManagerClassName()” is null
	at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) ~[?:?]
	at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1194) ~[?:?]
	at java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309) ~[?:?]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.selectAsync(ExtensibleLoadManagerImpl.java:385) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.lambda$assign$6(ExtensibleLoadManagerImpl.java:336) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187) ~[?:?]
	at java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309) ~[?:?]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.lambda$assign$10(ExtensibleLoadManagerImpl.java:333) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap$Section.put(ConcurrentOpenHashMap.java:409) ~[io.streamnative-pulsar-common-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap.computeIfAbsent(ConcurrentOpenHashMap.java:243) ~[io.streamnative-pulsar-common-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.assign(ExtensibleLoadManagerImpl.java:327) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper.findBrokerServiceUrl(ExtensibleLoadManagerWrapper.java:66) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
	at org.apache.pulsar.broker.namespace.NamespaceService.lambda$getBrokerServiceUrlAsync$0(NamespaceService.java:191) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1]
```

### Modifications

* Add null check when using`getLoadManagerClassName`.
* Add test to cover this case.
* Add `RedirectManager` unit test.
---
 .../filter/BrokerLoadManagerClassFilter.java  |   5 +-
 .../extensions/manager/RedirectManager.java   |  17 ++-
 .../impl/BrokerLoadManagerClassFilter.java    |   5 +-
 .../BrokerLoadManagerClassFilterTest.java     |   3 +-
 .../manager/RedirectManagerTest.java          | 111 ++++++++++++++++++
 .../BrokerLoadManagerClassFilterTest.java     |   6 +
 6 files changed, 139 insertions(+), 8 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
index 4ee28a5225a0d..07109b277ae98 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.filter;
 
 import java.util.Map;
+import java.util.Objects;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
@@ -43,7 +44,9 @@ public Map filter(
         }
         brokers.entrySet().removeIf(entry -> {
             BrokerLookupData v = entry.getValue();
-            return !v.getLoadManagerClassName().equals(context.brokerConfiguration().getLoadManagerClassName());
+            // The load manager class name can be null if the cluster has old version of broker.
+            return !Objects.equals(v.getLoadManagerClassName(),
+                    context.brokerConfiguration().getLoadManagerClassName());
         });
         return brokers;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
index 4aff77937a5b4..3455b333b0ae7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java
@@ -19,9 +19,11 @@
 package org.apache.pulsar.broker.loadbalance.extensions.manager;
 
 import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT;
+import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -48,6 +50,12 @@ public RedirectManager(PulsarService pulsar) {
         this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class);
     }
 
+    @VisibleForTesting
+    public RedirectManager(PulsarService pulsar, LockManager brokerLookupDataLockManager) {
+        this.pulsar = pulsar;
+        this.brokerLookupDataLockManager = brokerLookupDataLockManager;
+    }
+
     public CompletableFuture> getAvailableBrokerLookupDataAsync() {
         return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenCompose(availableBrokers -> {
             Map map = new ConcurrentHashMap<>();
@@ -69,7 +77,7 @@ public CompletableFuture> getAvailableBrokerLookup
 
     public CompletableFuture> findRedirectLookupResultAsync() {
         String currentLMClassName = pulsar.getConfiguration().getLoadManagerClassName();
-        boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfig(), log);
+        boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log);
         return getAvailableBrokerLookupDataAsync().thenApply(lookupDataMap -> {
             if (lookupDataMap.isEmpty()) {
                 String errorMsg = "No available broker found.";
@@ -89,9 +97,10 @@ public CompletableFuture> findRedirectLookupResultAsync()
                 log.warn(errorMsg);
                 throw new IllegalStateException(errorMsg);
             }
-            if (latestServiceLookupData.get().getLoadManagerClassName().equals(currentLMClassName)) {
+
+            if (Objects.equals(latestServiceLookupData.get().getLoadManagerClassName(), currentLMClassName)) {
                 if (debug) {
-                    log.info("We don't need to redirect, current load manager class name: {}",
+                    log.info("No need to redirect, current load manager class name: {}",
                             currentLMClassName);
                 }
                 return Optional.empty();
@@ -99,7 +108,7 @@ public CompletableFuture> findRedirectLookupResultAsync()
             var serviceLookupDataObj = latestServiceLookupData.get();
             var candidateBrokers = new ArrayList();
             lookupDataMap.forEach((key, value) -> {
-                if (value.getLoadManagerClassName().equals(serviceLookupDataObj.getLoadManagerClassName())) {
+                if (Objects.equals(value.getLoadManagerClassName(), serviceLookupDataObj.getLoadManagerClassName())) {
                     candidateBrokers.add(value);
                 }
             });
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
index 5d6a56ba86960..13e3fdc537e79 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.loadbalance.impl;
 
+import java.util.Objects;
 import java.util.Set;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilter;
@@ -32,8 +33,8 @@ public void filter(Set brokers, BundleData bundleToAssign,
                        LoadData loadData,
                        ServiceConfiguration conf) throws BrokerFilterException {
         loadData.getBrokerData().forEach((key, value) -> {
-            if (!value.getLocalData().getLoadManagerClassName()
-                    .equals(conf.getLoadManagerClassName())) {
+            // The load manager class name can be null if the cluster has old version of broker.
+            if (!Objects.equals(value.getLocalData().getLoadManagerClassName(), conf.getLoadManagerClassName())) {
                 brokers.remove(key);
             }
         });
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
index 0169b57fe993e..4aef87cf63aa8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java
@@ -44,7 +44,8 @@ public void test() throws BrokerFilterException {
                 "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
                 "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()),
                 "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()),
-                "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName())
+                "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()),
+                "broker5", getLookupData("3.0.0", null)
         );
 
         Map result = filter.filter(new HashMap<>(originalBrokers), null, context);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java
new file mode 100644
index 0000000000000..cbf77b59d5ad6
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
+import org.apache.pulsar.broker.lookup.LookupResult;
+import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
+import org.testng.annotations.Test;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+
+/**
+ * Unit test {@link RedirectManager}.
+ */
+public class RedirectManagerTest {
+
+    @Test
+    public void testFindRedirectLookupResultAsync() throws ExecutionException, InterruptedException {
+        PulsarService pulsar = mock(PulsarService.class);
+        ServiceConfiguration configuration = new ServiceConfiguration();
+        when(pulsar.getConfiguration()).thenReturn(configuration);
+        RedirectManager redirectManager = spy(new RedirectManager(pulsar, null));
+
+        configuration.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
+        configuration.setLoadBalancerDebugModeEnabled(true);
+
+        // Test 1: No load manager class name found.
+        doReturn(CompletableFuture.completedFuture(
+                new HashMap<>(){{
+                    put("broker-1", getLookupData("broker-1", null, 10));
+                    put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 1));
+                }}
+        )).when(redirectManager).getAvailableBrokerLookupDataAsync();
+
+        // Should redirect to broker-1, since broker-1 has the latest load manager, even though the class name is null.
+        Optional lookupResult = redirectManager.findRedirectLookupResultAsync().get();
+        assertTrue(lookupResult.isPresent());
+        assertTrue(lookupResult.get().getLookupData().getBrokerUrl().contains("broker-1"));
+
+        // Test 2: Should redirect to broker-1, since the latest broker are using ExtensibleLoadManagerImpl
+        doReturn(CompletableFuture.completedFuture(
+                new HashMap<>(){{
+                    put("broker-1", getLookupData("broker-1", ExtensibleLoadManagerImpl.class.getName(), 10));
+                    put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 1));
+                }}
+        )).when(redirectManager).getAvailableBrokerLookupDataAsync();
+
+        lookupResult = redirectManager.findRedirectLookupResultAsync().get();
+        assertTrue(lookupResult.isPresent());
+        assertTrue(lookupResult.get().getLookupData().getBrokerUrl().contains("broker-1"));
+
+
+        // Test 3: Should not redirect, since current broker are using ModularLoadManagerImpl
+        doReturn(CompletableFuture.completedFuture(
+                new HashMap<>(){{
+                    put("broker-1", getLookupData("broker-1", ExtensibleLoadManagerImpl.class.getName(), 10));
+                    put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 100));
+                }}
+        )).when(redirectManager).getAvailableBrokerLookupDataAsync();
+
+        lookupResult = redirectManager.findRedirectLookupResultAsync().get();
+        assertFalse(lookupResult.isPresent());
+    }
+
+
+    public BrokerLookupData getLookupData(String broker, String loadManagerClassName, long startTimeStamp) {
+        String webServiceUrl = "http://" + broker + ":8080";
+        String webServiceUrlTls = "https://" + broker + ":8081";
+        String pulsarServiceUrl = "pulsar://" + broker + ":6650";
+        String pulsarServiceUrlTls = "pulsar+ssl://" + broker + ":6651";
+        Map advertisedListeners = new HashMap<>();
+        Map protocols = new HashMap<>(){{
+            put("kafka", "9092");
+        }};
+        return new BrokerLookupData(
+                webServiceUrl, webServiceUrlTls, pulsarServiceUrl,
+                pulsarServiceUrlTls, advertisedListeners, protocols, true, true,
+                loadManagerClassName, startTimeStamp, "3.0.0");
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
index 856bbac029226..56332111f935b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java
@@ -46,8 +46,12 @@ public void test() throws BrokerFilterException {
 
         LocalBrokerData localBrokerData1 = new LocalBrokerData();
         localBrokerData1.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
+
+        LocalBrokerData localBrokerData2 = new LocalBrokerData();
+        localBrokerData2.setLoadManagerClassName(null);
         loadData.getBrokerData().put("broker1", new BrokerData(localBrokerData));
         loadData.getBrokerData().put("broker2", new BrokerData(localBrokerData1));
+        loadData.getBrokerData().put("broker3", new BrokerData(localBrokerData2));
 
         ServiceConfiguration conf = new ServiceConfiguration();
         conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
@@ -55,6 +59,7 @@ public void test() throws BrokerFilterException {
         Set brokers = new HashSet<>(){{
             add("broker1");
             add("broker2");
+            add("broker3");
         }};
         filter.filter(brokers, null, loadData, conf);
 
@@ -64,6 +69,7 @@ public void test() throws BrokerFilterException {
         brokers = new HashSet<>(){{
             add("broker1");
             add("broker2");
+            add("broker3");
         }};
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         filter.filter(brokers, null, loadData, conf);

From 548fd22b32bef8d64b4733954de0e2945d3895b0 Mon Sep 17 00:00:00 2001
From: Paul Gier 
Date: Mon, 22 May 2023 14:31:12 -0500
Subject: [PATCH 305/404] [improve][cli] pulsar-perf: refactor to reduce code
 duplication (#19279)

---
 .../pulsar/testclient/PerfClientUtils.java    |   1 -
 .../testclient/PerformanceBaseArguments.java  |  69 +++++++++++-
 .../testclient/PerformanceConsumer.java       | 105 ++++--------------
 .../testclient/PerformanceProducer.java       |  61 +---------
 .../pulsar/testclient/PerformanceReader.java  |  67 +++--------
 .../PerformanceTopicListArguments.java        |  67 +++++++++++
 .../testclient/PerformanceTransaction.java    |  24 +---
 .../testclient/PerfClientUtilsTest.java       |  32 ++----
 .../PerformanceBaseArgumentsTest.java         |   3 +-
 9 files changed, 186 insertions(+), 243 deletions(-)
 create mode 100644 pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java

diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java
index e40e9610bf42f..f9e5d5ee7e6e1 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java
@@ -119,5 +119,4 @@ public static PulsarAdminBuilder createAdminBuilderFromArguments(PerformanceBase
         return pulsarAdminBuilder;
     }
 
-
 }
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java
index ce402884d5ced..5ae79fb0bf9a4 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java
@@ -20,20 +20,26 @@
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.pulsar.testclient.PerfClientUtils.exit;
+import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import java.io.File;
 import java.io.FileInputStream;
 import java.util.Properties;
 import lombok.SneakyThrows;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.client.api.ProxyProtocol;
 
-
+/**
+ * PerformanceBaseArguments contains common CLI arguments and parsing logic available to all sub-commands.
+ * Sub-commands should create Argument subclasses and override the `validate` method as necessary.
+ */
 public abstract class PerformanceBaseArguments {
 
-    @Parameter(names = { "-h", "--help" }, description = "Help message", help = true)
+    @Parameter(names = { "-h", "--help" }, description = "Print help message", help = true)
     boolean help;
 
-    @Parameter(names = { "-cf", "--conf-file" }, description = "Configuration file")
+    @Parameter(names = { "-cf", "--conf-file" }, description = "Pulsar configuration file")
     public String confFile;
 
     @Parameter(names = { "-u", "--service-url" }, description = "Pulsar Service URL")
@@ -94,6 +100,9 @@ public abstract class PerformanceBaseArguments {
     @Parameter(names = { "--proxy-protocol" }, description = "Proxy protocol to select type of routing at proxy.")
     ProxyProtocol proxyProtocol = null;
 
+    @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true)
+    public String deprecatedAuthPluginClassName;
+
     public abstract void fillArgumentsFromProperties(Properties prop);
 
     @SneakyThrows
@@ -165,4 +174,58 @@ public void fillArgumentsFromProperties() {
         fillArgumentsFromProperties(prop);
     }
 
+    /**
+     * Validate the CLI arguments.  Default implementation provides validation for the common arguments.
+     * Each subclass should call super.validate() and provide validation code specific to the sub-command.
+     * @throws Exception
+     */
+    public void validate() throws Exception {
+        if (confFile != null && !confFile.isBlank()) {
+            File configFile = new File(confFile);
+            if (!configFile.exists()) {
+                throw new Exception("config file '" + confFile + "', does not exist");
+            }
+            if (configFile.isDirectory()) {
+                throw new Exception("config file '" + confFile + "', is a directory");
+            }
+        }
+    }
+
+    /**
+     * Parse the command line args.
+     * @param cmdName used for the help message
+     * @param args String[] of CLI args
+     * @throws ParameterException If there is a problem parsing the arguments
+     */
+    public void parseCLI(String cmdName, String[] args) {
+        JCommander jc = new JCommander(this);
+        jc.setProgramName(cmdName);
+
+        try {
+            jc.parse(args);
+        } catch (ParameterException e) {
+            System.out.println("error: " + e.getMessage());
+            jc.usage();
+            PerfClientUtils.exit(1);
+        }
+
+        if (help) {
+            jc.usage();
+            PerfClientUtils.exit(0);
+        }
+
+        fillArgumentsFromProperties();
+
+        if (isBlank(authPluginClassName) && !isBlank(deprecatedAuthPluginClassName)) {
+            authPluginClassName = deprecatedAuthPluginClassName;
+        }
+
+        try {
+            validate();
+        } catch (Exception e) {
+            System.out.println("error: " + e.getMessage());
+            PerfClientUtils.exit(1);
+        }
+    }
+
 }
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
index 11a23ca05e0b1..59dabc9302622 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
@@ -18,11 +18,8 @@
  */
 package org.apache.pulsar.testclient;
 
-import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -86,14 +83,7 @@ public class PerformanceConsumer {
     private static final Recorder cumulativeRecorder = new Recorder(MAX_LATENCY, 5);
 
     @Parameters(commandDescription = "Test pulsar consumer performance.")
-    static class Arguments extends PerformanceBaseArguments {
-
-        @Parameter(description = "persistent://prop/ns/my-topic", required = true)
-        public List topic;
-
-        @Parameter(names = { "-t", "--num-topics" }, description = "Number of topics",
-                validateWith = PositiveNumberParameterValidator.class)
-        public int numTopics = 1;
+    static class Arguments extends PerformanceTopicListArguments {
 
         @Parameter(names = { "-n", "--num-consumers" }, description = "Number of consumers (per subscription), only "
                 + "one consumer is allowed when subscriptionType is Exclusive",
@@ -143,9 +133,6 @@ static class Arguments extends PerformanceBaseArguments {
                 description = "Number of messages to consume in total. If <= 0, it will keep consuming")
         public long numMessages = 0;
 
-        @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true)
-        public String deprecatedAuthPluginClassName;
-
         @Parameter(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages")
         private int maxPendingChunkedMessage = 0;
 
@@ -199,79 +186,35 @@ static class Arguments extends PerformanceBaseArguments {
         @Override
         public void fillArgumentsFromProperties(Properties prop) {
         }
-    }
-
-    public static void main(String[] args) throws Exception {
-        final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf consume");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
 
-        if (isBlank(arguments.authPluginClassName) && !isBlank(arguments.deprecatedAuthPluginClassName)) {
-            arguments.authPluginClassName = arguments.deprecatedAuthPluginClassName;
-        }
-
-        for (String arg : arguments.topic) {
-            if (arg.startsWith("-")) {
-                System.out.printf("invalid option: '%s'\nTo use a topic with the name '%s', "
-                        + "please use a fully qualified topic name\n", arg, arg);
-                jc.usage();
-                PerfClientUtils.exit(1);
+        @Override
+        public void validate() throws Exception {
+            super.validate();
+            if (subscriptionType == SubscriptionType.Exclusive && numConsumers > 1) {
+                throw new Exception("Only one consumer is allowed when subscriptionType is Exclusive");
             }
-        }
 
-        if (arguments.topic != null && arguments.topic.size() != arguments.numTopics) {
-            // keep compatibility with the previous version
-            if (arguments.topic.size() == 1) {
-                String prefixTopicName = TopicName.get(arguments.topic.get(0)).toString().trim();
-                List defaultTopics = new ArrayList<>();
-                for (int i = 0; i < arguments.numTopics; i++) {
-                    defaultTopics.add(String.format("%s-%d", prefixTopicName, i));
+            if (subscriptions != null && subscriptions.size() != numSubscriptions) {
+                // keep compatibility with the previous version
+                if (subscriptions.size() == 1) {
+                    if (subscriberName == null) {
+                        subscriberName = subscriptions.get(0);
+                    }
+                    List defaultSubscriptions = new ArrayList<>();
+                    for (int i = 0; i < numSubscriptions; i++) {
+                        defaultSubscriptions.add(String.format("%s-%d", subscriberName, i));
+                    }
+                    subscriptions = defaultSubscriptions;
+                } else {
+                    throw new Exception("The size of subscriptions list should be equal to --num-subscriptions");
                 }
-                arguments.topic = defaultTopics;
-            } else {
-                System.out.println("The size of topics list should be equal to --num-topics");
-                jc.usage();
-                PerfClientUtils.exit(1);
             }
         }
+    }
 
-        if (arguments.subscriptionType == SubscriptionType.Exclusive && arguments.numConsumers > 1) {
-            System.out.println("Only one consumer is allowed when subscriptionType is Exclusive");
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.subscriptions != null && arguments.subscriptions.size() != arguments.numSubscriptions) {
-            // keep compatibility with the previous version
-            if (arguments.subscriptions.size() == 1) {
-                if (arguments.subscriberName == null) {
-                    arguments.subscriberName = arguments.subscriptions.get(0);
-                }
-                List defaultSubscriptions = new ArrayList<>();
-                for (int i = 0; i < arguments.numSubscriptions; i++) {
-                    defaultSubscriptions.add(String.format("%s-%d", arguments.subscriberName, i));
-                }
-                arguments.subscriptions = defaultSubscriptions;
-            } else {
-                System.out.println("The size of subscriptions list should be equal to --num-subscriptions");
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-        arguments.fillArgumentsFromProperties();
+    public static void main(String[] args) throws Exception {
+        final Arguments arguments = new Arguments();
+        arguments.parseCLI("pulsar-perf consume", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
@@ -449,7 +392,7 @@ public static void main(String[] args) throws Exception {
         }
 
         for (int i = 0; i < arguments.numTopics; i++) {
-            final TopicName topicName = TopicName.get(arguments.topic.get(i));
+            final TopicName topicName = TopicName.get(arguments.topics.get(i));
 
             log.info("Adding {} consumers per subscription on topic {}", arguments.numConsumers, topicName);
 
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index 0c56f1d5c736d..6513f0684b243 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -24,9 +24,7 @@
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES;
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES;
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -103,10 +101,7 @@ public class PerformanceProducer {
     private static IMessageFormatter messageFormatter = null;
 
     @Parameters(commandDescription = "Test pulsar producer performance.")
-    static class Arguments extends PerformanceBaseArguments {
-
-        @Parameter(description = "persistent://prop/ns/my-topic", required = true)
-        public List topics;
+    static class Arguments extends PerformanceTopicListArguments {
 
         @Parameter(names = { "-threads", "--num-test-threads" }, description = "Number of test threads",
                 validateWith = PositiveNumberParameterValidator.class)
@@ -118,10 +113,6 @@ static class Arguments extends PerformanceBaseArguments {
         @Parameter(names = { "-s", "--size" }, description = "Message size (bytes)")
         public int msgSize = 1024;
 
-        @Parameter(names = { "-t", "--num-topic" }, description = "Number of topics",
-                validateWith = PositiveNumberParameterValidator.class)
-        public int numTopics = 1;
-
         @Parameter(names = { "-n", "--num-producers" }, description = "Number of producers (per topic)",
                 validateWith = PositiveNumberParameterValidator.class)
         public int numProducers = 1;
@@ -139,9 +130,6 @@ static class Arguments extends PerformanceBaseArguments {
         @Parameter(names = { "-au", "--admin-url" }, description = "Pulsar Admin URL")
         public String adminURL;
 
-        @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true)
-        public String deprecatedAuthPluginClassName;
-
         @Parameter(names = { "-ch",
                 "--chunking" }, description = "Should split the message and publish in chunks if message size is "
                 + "larger than allowed max size")
@@ -272,52 +260,7 @@ public void fillArgumentsFromProperties(Properties prop) {
     public static void main(String[] args) throws Exception {
 
         final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf produce");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (isBlank(arguments.authPluginClassName) && !isBlank(arguments.deprecatedAuthPluginClassName)) {
-            arguments.authPluginClassName = arguments.deprecatedAuthPluginClassName;
-        }
-
-        for (String arg : arguments.topics) {
-            if (arg.startsWith("-")) {
-                System.out.printf("invalid option: '%s'\nTo use a topic with the name '%s', "
-                        + "please use a fully qualified topic name\n", arg, arg);
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-
-        if (arguments.topics != null && arguments.topics.size() != arguments.numTopics) {
-            // keep compatibility with the previous version
-            if (arguments.topics.size() == 1) {
-                String prefixTopicName = arguments.topics.get(0);
-                List defaultTopics = new ArrayList<>();
-                for (int i = 0; i < arguments.numTopics; i++) {
-                    defaultTopics.add(String.format("%s%s%d", prefixTopicName, arguments.separator, i));
-                }
-                arguments.topics = defaultTopics;
-            } else {
-                System.out.println("The size of topics list should be equal to --num-topic");
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-
-        arguments.fillArgumentsFromProperties();
+        arguments.parseCLI("pulsar-perf produce", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java
index 78d8e5f591569..ed5cc37644a31 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java
@@ -18,9 +18,7 @@
  */
 package org.apache.pulsar.testclient;
 
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -62,15 +60,7 @@ public class PerformanceReader {
     private static Recorder cumulativeRecorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5);
 
     @Parameters(commandDescription = "Test pulsar reader performance.")
-    static class Arguments extends PerformanceBaseArguments {
-
-
-        @Parameter(description = "persistent://prop/ns/my-topic", required = true)
-        public List topic;
-
-        @Parameter(names = { "-t", "--num-topics" }, description = "Number of topics",
-                validateWith = PositiveNumberParameterValidator.class)
-        public int numTopics = 1;
+    static class Arguments extends PerformanceTopicListArguments {
 
         @Parameter(names = { "-r", "--rate" }, description = "Simulate a slow message reader (rate in msg/s)")
         public double rate = 0;
@@ -102,51 +92,21 @@ public void fillArgumentsFromProperties(Properties prop) {
                 useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
             }
         }
+        @Override
+        public void validate() throws Exception {
+            super.validate();
+            if (startMessageId != "earliest" && startMessageId != "latest"
+                    && (startMessageId.split(":")).length != 2) {
+                String errMsg = String.format("invalid start message ID '%s', must be either either 'earliest', "
+                        + "'latest' or a specific message id by using 'lid:eid'", startMessageId);
+                throw new Exception(errMsg);
+            }
+        }
     }
 
     public static void main(String[] args) throws Exception {
         final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf read");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        for (String arg : arguments.topic) {
-            if (arg.startsWith("-")) {
-                System.out.printf("invalid option: '%s'\nTo use a topic with the name '%s', "
-                        + "please use a fully qualified topic name\n", arg, arg);
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-
-        if (arguments.topic != null && arguments.topic.size() != arguments.numTopics) {
-            // keep compatibility with the previous version
-            if (arguments.topic.size() == 1) {
-                String prefixTopicName = arguments.topic.get(0);
-                List defaultTopics = new ArrayList<>();
-                for (int i = 0; i < arguments.numTopics; i++) {
-                    defaultTopics.add(String.format("%s-%d", prefixTopicName, i));
-                }
-                arguments.topic = defaultTopics;
-            } else {
-                System.out.println("The size of topics list should be equal to --num-topics");
-                jc.usage();
-                PerfClientUtils.exit(1);
-            }
-        }
-        arguments.fillArgumentsFromProperties();
+        arguments.parseCLI("pulsar-perf read", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
@@ -202,7 +162,7 @@ public static void main(String[] args) throws Exception {
                 .startMessageId(startMessageId);
 
         for (int i = 0; i < arguments.numTopics; i++) {
-            final TopicName topicName = TopicName.get(arguments.topic.get(i));
+            final TopicName topicName = TopicName.get(arguments.topics.get(i));
 
             futures.add(readerBuilder.clone().topic(topicName.toString()).createAsync());
         }
@@ -230,7 +190,6 @@ public void run() {
             timer.schedule(timoutTask, arguments.testTime * 1000);
         }
 
-
         long oldTime = System.nanoTime();
         Histogram reportHistogram = null;
 
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java
new file mode 100644
index 0000000000000..a2f8b6af08282
--- /dev/null
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.testclient;
+
+import com.beust.jcommander.Parameter;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pulsar.common.naming.TopicName;
+
+/**
+ * PerformanceTopicListArguments provides common topic list arguments which are used
+ * by the consumer, producer, and reader commands, but not by the transaction test command.
+ */
+public abstract class PerformanceTopicListArguments extends PerformanceBaseArguments {
+
+    @Parameter(description = "persistent://prop/ns/my-topic", required = true)
+    public List topics;
+
+    @Parameter(names = { "-t", "--num-topics", "--num-topic" }, description = "Number of topics.  Must match"
+            + "the given number of topic arguments.",
+            validateWith = PositiveNumberParameterValidator.class)
+    public int numTopics = 1;
+
+    @Override
+    public void validate() throws Exception {
+        super.validate();
+        for (String arg : topics) {
+            if (arg.startsWith("-")) {
+                String errMsg = String.format("invalid option: '%s', to use a topic with the name '%s', "
+                        + "please use a fully qualified topic name", arg, arg);
+                throw new Exception(errMsg);
+            }
+        }
+
+        if (topics.size() != numTopics) {
+            // keep compatibility with the previous version
+            if (topics.size() == 1) {
+                String prefixTopicName = TopicName.get(topics.get(0)).toString().trim();
+                List defaultTopics = new ArrayList<>();
+                for (int i = 0; i < numTopics; i++) {
+                    defaultTopics.add(String.format("%s-%d", prefixTopicName, i));
+                }
+                topics = defaultTopics;
+            } else {
+                String errMsg = String.format("the number of topic names (%d) must be equal to --num-topics (%d)",
+                        topics.size(), numTopics);
+                throw new Exception(errMsg);
+            }
+        }
+    }
+}
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java
index a1495a617fb9c..469e6ab1f3fd6 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java
@@ -19,9 +19,7 @@
 package org.apache.pulsar.testclient;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -70,7 +68,6 @@
 
 public class PerformanceTransaction {
 
-
     private static final LongAdder totalNumEndTxnOpFailed = new LongAdder();
     private static final LongAdder totalNumEndTxnOpSuccess = new LongAdder();
     private static final LongAdder numTxnOpSuccess = new LongAdder();
@@ -92,9 +89,8 @@ public class PerformanceTransaction {
     private static final Recorder messageSendRCumulativeRecorder =
             new Recorder(TimeUnit.SECONDS.toMicros(120000), 5);
 
-
     @Parameters(commandDescription = "Test pulsar transaction performance.")
-    static class Arguments  extends PerformanceBaseArguments {
+    static class Arguments extends PerformanceBaseArguments {
 
         @Parameter(names = "--topics-c", description = "All topics that need ack for a transaction", required =
                 true)
@@ -187,26 +183,10 @@ public void fillArgumentsFromProperties(Properties prop) {
     public static void main(String[] args)
             throws IOException, PulsarAdminException, ExecutionException, InterruptedException {
         final Arguments arguments = new Arguments();
-        JCommander jc = new JCommander(arguments);
-        jc.setProgramName("pulsar-perf transaction");
-
-        try {
-            jc.parse(args);
-        } catch (ParameterException e) {
-            System.out.println(e.getMessage());
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-
-        if (arguments.help) {
-            jc.usage();
-            PerfClientUtils.exit(1);
-        }
-        arguments.fillArgumentsFromProperties();
+        arguments.parseCLI("pulsar-perf transaction", args);
 
         // Dump config variables
         PerfClientUtils.printJVMInformation(log);
-
         ObjectMapper m = new ObjectMapper();
         ObjectWriter w = m.writerWithDefaultPrettyPrinter();
         log.info("Starting Pulsar perf transaction with config: {}", w.writeValueAsString(arguments));
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java
index 3a8bba1bccb90..3d734b1f910ea 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java
@@ -55,11 +55,7 @@ public void close() throws IOException {
     @Test
     public void testClientCreation() throws Exception {
 
-        final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-            @Override
-            public void fillArgumentsFromProperties(Properties prop) {
-            }
-        };
+        final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
         args.tlsHostnameVerificationEnable = true;
         args.authPluginClassName = MyAuth.class.getName();
@@ -99,11 +95,7 @@ public void fillArgumentsFromProperties(Properties prop) {
     @Test
     public void testClientCreationWithProxy() throws Exception {
 
-        final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-            @Override
-            public void fillArgumentsFromProperties(Properties prop) {
-            }
-        };
+        final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
         args.serviceURL = "pulsar+ssl://my-pulsar:6651";
         args.proxyServiceURL = "pulsar+ssl://my-proxy-pulsar:4443";
@@ -126,11 +118,7 @@ public void testClientCreationWithProxyDefinedInConfFile() throws Exception {
                     + "proxyServiceUrl=pulsar+ssl://my-proxy-pulsar:4443\n"
                     + "proxyProtocol=SNI");
 
-            final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-                @Override
-                public void fillArgumentsFromProperties(Properties prop) {
-                }
-            };
+            final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
             args.confFile = testConf.toString();
             args.fillArgumentsFromProperties();
@@ -155,11 +143,7 @@ public void testClientCreationWithEmptyProxyPropertyInConfFile() throws Exceptio
                     + "proxyServiceUrl=\n"
                     + "proxyProtocol=");
 
-            final PerformanceBaseArguments args = new PerformanceBaseArguments() {
-                @Override
-                public void fillArgumentsFromProperties(Properties prop) {
-                }
-            };
+            final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault();
 
             args.confFile = testConf.toString();
             args.fillArgumentsFromProperties();
@@ -174,4 +158,10 @@ public void fillArgumentsFromProperties(Properties prop) {
             Files.deleteIfExists(testConf);
         }
     }
-}
\ No newline at end of file
+}
+
+class PerformanceArgumentsTestDefault extends PerformanceBaseArguments {
+    @Override
+    public void fillArgumentsFromProperties(Properties prop) {
+    }
+}
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java
index 6c60cbd90f8d5..42c93be343074 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java
@@ -158,5 +158,4 @@ public void fillArgumentsFromProperties(Properties prop) {
             tempConfigFile.delete();
         }
     }
-
-}
\ No newline at end of file
+}

From e008de9466f452dd92997a11621df3e30b024ece Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 23 May 2023 22:11:30 +0300
Subject: [PATCH 306/404] [fix][broker] Change some static fields referencing
 mutable objects to ordinary instance fields in BrokerService (#20372)

---
 .../apache/pulsar/broker/admin/impl/BrokersBase.java |  9 ++++-----
 .../apache/pulsar/broker/service/BrokerService.java  | 12 ++++++------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
index 3328fa9715b99..b367ce7aad955 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
@@ -53,7 +53,6 @@
 import org.apache.pulsar.broker.admin.AdminResource;
 import org.apache.pulsar.broker.loadbalance.LeaderBroker;
 import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.web.RestException;
@@ -254,7 +253,7 @@ public void getAllDynamicConfigurations(@Suspended AsyncResponse asyncResponse)
             @ApiResponse(code = 403, message = "You don't have admin permission to get configuration")})
     public void getDynamicConfigurationName(@Suspended AsyncResponse asyncResponse) {
         validateSuperUserAccessAsync()
-                .thenAccept(__ -> asyncResponse.resume(BrokerService.getDynamicConfiguration()))
+                .thenAccept(__ -> asyncResponse.resume(pulsar().getBrokerService().getDynamicConfiguration()))
                 .exceptionally(ex -> {
                     LOG.error("[{}] Failed to get all dynamic configuration names.", clientAppId(), ex);
                     resumeAsyncResponseExceptionally(asyncResponse, ex);
@@ -287,11 +286,11 @@ public void getRuntimeConfiguration(@Suspended AsyncResponse asyncResponse) {
      */
     private synchronized CompletableFuture persistDynamicConfigurationAsync(
             String configName, String configValue) {
-        if (!BrokerService.validateDynamicConfiguration(configName, configValue)) {
+        if (!pulsar().getBrokerService().validateDynamicConfiguration(configName, configValue)) {
             return FutureUtil
                     .failedFuture(new RestException(Status.PRECONDITION_FAILED, " Invalid dynamic-config value"));
         }
-        if (BrokerService.isDynamicConfiguration(configName)) {
+        if (pulsar().getBrokerService().isDynamicConfiguration(configName)) {
             return dynamicConfigurationResources().setDynamicConfigurationWithCreateAsync(old -> {
                 Map configurationMap = old.orElseGet(Maps::newHashMap);
                 configurationMap.put(configName, configValue);
@@ -512,7 +511,7 @@ private CompletableFuture healthCheckRecursiveReadNext(Reader read
     }
 
     private CompletableFuture internalDeleteDynamicConfigurationOnMetadataAsync(String configName) {
-        if (!BrokerService.isDynamicConfiguration(configName)) {
+        if (!pulsar().getBrokerService().isDynamicConfiguration(configName)) {
             throw new RestException(Status.PRECONDITION_FAILED, " Can't update non-dynamic configuration");
         } else {
             return dynamicConfigurationResources().setDynamicConfigurationAsync(old -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 33e5500d623d2..663d013dc7439 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -214,7 +214,7 @@ public class BrokerService implements Closeable {
     private final OrderedExecutor topicOrderedExecutor;
     // offline topic backlog cache
     private final ConcurrentOpenHashMap offlineTopicStatCache;
-    private static final ConcurrentOpenHashMap dynamicConfigurationMap =
+    private final ConcurrentOpenHashMap dynamicConfigurationMap =
             prepareDynamicConfigurationMap();
     private final ConcurrentOpenHashMap> configRegisteredListeners;
 
@@ -253,10 +253,10 @@ public class BrokerService implements Closeable {
 
     public static final String MANAGED_LEDGER_PATH_ZNODE = "/managed-ledgers";
 
-    private static final LongAdder totalUnackedMessages = new LongAdder();
+    private final LongAdder totalUnackedMessages = new LongAdder();
     private final int maxUnackedMessages;
     public final int maxUnackedMsgsPerDispatcher;
-    private static final AtomicBoolean blockedDispatcherOnHighUnackedMsgs = new AtomicBoolean(false);
+    private final AtomicBoolean blockedDispatcherOnHighUnackedMsgs = new AtomicBoolean(false);
     private final ConcurrentOpenHashSet blockedDispatchers;
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
@@ -2938,7 +2938,7 @@ public DelayedDeliveryTrackerFactory getDelayedDeliveryTrackerFactory() {
         return delayedDeliveryTrackerFactory;
     }
 
-    public static List getDynamicConfiguration() {
+    public List getDynamicConfiguration() {
         return dynamicConfigurationMap.keys();
     }
 
@@ -2951,11 +2951,11 @@ public Map getRuntimeConfiguration() {
         return configMap;
     }
 
-    public static boolean isDynamicConfiguration(String key) {
+    public boolean isDynamicConfiguration(String key) {
         return dynamicConfigurationMap.containsKey(key);
     }
 
-    public static boolean validateDynamicConfiguration(String key, String value) {
+    public boolean validateDynamicConfiguration(String key, String value) {
         if (dynamicConfigurationMap.containsKey(key) && dynamicConfigurationMap.get(key).validator != null) {
             return dynamicConfigurationMap.get(key).validator.test(value);
         }

From 120f229ffd4a2ce264b3a95a6731d948e2d88323 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 23 May 2023 14:19:12 -0500
Subject: [PATCH 307/404] [fix] Increase timeout on
 ManagedLedgerCompressionTest flaky test (#20352)

---
 .../pulsar/broker/service/ManagedLedgerCompressionTest.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
index 1ecb0d8f5f7a0..fb60ef97320fb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
@@ -50,7 +50,7 @@ protected void cleanup() throws Exception {
         super.internalCleanup();
     }
 
-    @Test(timeOut = 1000 * 30)
+    @Test(timeOut = 1000 * 60)
     public void testRestartBrokerEnableManagedLedgerInfoCompression() throws Exception {
         String topic = newTopicName();
         @Cleanup

From 2e6928a02037087e3809fe35c55fd738bb229ee7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Wed, 24 May 2023 03:36:10 +0200
Subject: [PATCH 308/404] [improve][client] Swallow Conscrypt
 ClassNotFoundException (#20371)

---
 .../java/org/apache/pulsar/common/util/SecurityUtility.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java
index 12ab9ae0b0bc9..f0023ce5a42dd 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java
@@ -124,7 +124,9 @@ private static Provider loadConscryptProvider() {
             conscryptClazz = Class.forName("org.conscrypt.Conscrypt");
             conscryptClazz.getMethod("checkAvailability").invoke(null);
         } catch (Throwable e) {
-            if (e.getCause() instanceof UnsatisfiedLinkError) {
+            if (e instanceof ClassNotFoundException) {
+                log.warn("Conscrypt isn't available in the classpath. Using JDK default security provider.");
+            } else if (e.getCause() instanceof UnsatisfiedLinkError) {
                 log.warn("Conscrypt isn't available for {} {}. Using JDK default security provider.",
                         System.getProperty("os.name"), System.getProperty("os.arch"));
             } else {

From 1080ad5c787bb317347b3f1f12b78ba3dec49757 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 23 May 2023 21:26:49 -0700
Subject: [PATCH 309/404] [fix][broker] pre-create non-partitioned system
 topics for load balance extension (#20370)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation

We need to create system topics without partitions explicitly. Currently, we do not support partitioned system topics.

### Modifications

 create system topics without partitions explicitly
---
 .../extensions/ExtensibleLoadManagerImpl.java   | 17 +++++++++++++++++
 .../channel/ServiceUnitStateChannelImpl.java    |  2 ++
 .../ExtensibleLoadManagerImplTest.java          |  2 ++
 .../channel/ServiceUnitStateChannelTest.java    |  2 ++
 4 files changed, 23 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index cbaed8ee8f94f..531ab18938a1e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -79,6 +79,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
+import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceName;
@@ -213,6 +214,19 @@ public static boolean debug(ServiceConfiguration config, Logger log) {
         return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
     }
 
+    public static void createSystemTopic(PulsarService pulsar, String topic) throws PulsarServerException {
+        try {
+            pulsar.getAdminClient().topics().createNonPartitionedTopic(topic);
+            log.info("Created topic {}.", topic);
+        } catch (PulsarAdminException.ConflictException ex) {
+            if (debug(pulsar.getConfiguration(), log)) {
+                log.info("Topic {} already exists.", topic, ex);
+            }
+        } catch (PulsarAdminException e) {
+            throw new PulsarServerException(e);
+        }
+    }
+
     @Override
     public void start() throws PulsarServerException {
         if (this.started) {
@@ -247,6 +261,9 @@ public void start() throws PulsarServerException {
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies);
         this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper));
 
+        createSystemTopic(pulsar, BROKER_LOAD_DATA_STORE_TOPIC);
+        createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC);
+
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 6246c26e57bcc..489a00851057b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -294,6 +294,8 @@ public synchronized void start() throws PulsarServerException {
             PulsarClusterMetadataSetup.createNamespaceIfAbsent
                     (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, config.getClusterName());
 
+            ExtensibleLoadManagerImpl.createSystemTopic(pulsar, TOPIC);
+
             producer = pulsar.getClient().newProducer(schema)
                     .enableBatching(true)
                     .compressionType(MSG_COMPRESSION_TYPE)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index b131aff68d909..498f48d16d2cd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -97,6 +97,7 @@
 import org.apache.pulsar.common.policies.data.BundlesData;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
@@ -129,6 +130,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest {
     @Override
     public void setup() throws Exception {
         conf.setForceDeleteNamespaceAllowed(true);
+        conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED);
         conf.setAllowAutoTopicCreation(true);
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index cb26c460f0a03..1263170bec495 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -86,6 +86,7 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.TypedMessageBuilder;
 import org.apache.pulsar.client.impl.TableViewImpl;
+import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.NotificationType;
@@ -129,6 +130,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest {
     @Override
     protected void setup() throws Exception {
         conf.setAllowAutoTopicCreation(true);
+        conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED);
         conf.setLoadBalancerDebugModeEnabled(true);
         conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10);
         super.internalSetup(conf);

From f2ab623095a54968859c1db041d1f24c54dee20a Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 24 May 2023 13:07:07 +0800
Subject: [PATCH 310/404] [fix][broker] Fix memory leak when rebatchMessage
 (#20369)

---
 .../java/org/apache/pulsar/client/impl/RawBatchConverter.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
index 4809ce1a04807..54d2ff867a629 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
@@ -100,7 +100,7 @@ public static Optional rebatchMessage(RawMessage msg,
             payload.skipBytes(Short.BYTES);
             int brokerEntryMetadataSize = payload.readInt();
             payload.readerIndex(readerIndex);
-            brokerMeta = payload.readRetainedSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES);
+            brokerMeta = payload.readSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES);
         }
         MessageMetadata metadata = Commands.parseMessageMetadata(payload);
         ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity());
@@ -152,7 +152,7 @@ public static Optional rebatchMessage(RawMessage msg,
 
                 if (brokerMeta != null) {
                     CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeDirectBuffer();
-                    compositeByteBuf.addComponents(true, brokerMeta, metadataAndPayload);
+                    compositeByteBuf.addComponents(true, brokerMeta.retain(), metadataAndPayload);
                     metadataAndPayload = compositeByteBuf;
                 }
 

From 0e7056bf91af3c8d10026d2f238b31d56c0c2130 Mon Sep 17 00:00:00 2001
From: GerMoranOverstock <126716444+GerMoranOverstock@users.noreply.github.com>
Date: Wed, 24 May 2023 06:31:36 +0100
Subject: [PATCH 311/404] [improve][doc] Remove javadoc reference to old
 default 30s ack timeout impl (#20377)

---
 .../main/java/org/apache/pulsar/client/api/ConsumerBuilder.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java
index 14a94cb8286dc..870900a48feae 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java
@@ -184,8 +184,6 @@ public interface ConsumerBuilder extends Cloneable {
      * 
    By default, the acknowledgment timeout is disabled (set to `0`, which means infinite).
      * When a consumer with an infinite acknowledgment timeout terminates, any unacknowledged
      * messages that it receives are re-delivered to another consumer.
-     * 
    Since 2.3.0, when a dead letter policy is specified and no ackTimeoutMillis is specified,
-     * the acknowledgment timeout is set to 30 seconds.
      *
      * 
    When enabling acknowledgment timeout, if a message is not acknowledged within the specified timeout,
      * it is re-delivered to the consumer (possibly to a different consumer, in the case of

From 946acc551da55d69a6d26d4696b4954cf89a88d2 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Wed, 24 May 2023 15:06:34 +0800
Subject: [PATCH 312/404] [improve][monitor] Add JVM start time metric (#20381)

---
 .../main/java/org/apache/pulsar/common/stats/JvmMetrics.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java
index 1f15beb8a0b92..8a8da0bb1ac93 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java
@@ -99,6 +99,9 @@ public List generate() {
 
         Runtime r = Runtime.getRuntime();
 
+        RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+
+        m.put("jvm_start_time", runtimeMXBean.getStartTime());
         m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
         m.put("jvm_max_memory", r.maxMemory());
         m.put("jvm_total_memory", r.totalMemory());

From 05e57dd3a443c5b99c21054c56a1b497455fa867 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Wed, 24 May 2023 16:41:52 +0800
Subject: [PATCH 313/404] [fix][fn] Fix JavaInstanceStarter inferring type
 class name error (#19896)

---
 .../runtime/JavaInstanceStarter.java          | 19 ++++++++++----
 .../runtime/thread/ThreadRuntime.java         | 19 ++++++++------
 .../runtime/thread/ThreadRuntimeFactory.java  | 26 ++++++++++++++++---
 3 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index c4f44be3df380..33e837d66e250 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -49,10 +49,13 @@
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.proto.InstanceControlGrpc;
+import org.apache.pulsar.functions.runtime.thread.ThreadRuntime;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory;
 import org.apache.pulsar.functions.secretsprovider.ClearTextSecretsProvider;
 import org.apache.pulsar.functions.secretsprovider.SecretsProvider;
 import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager;
+import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManagerImpl;
 
 
 @Slf4j
@@ -192,7 +195,10 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL
             functionDetailsJsonString = functionDetailsJsonString.substring(0, functionDetailsJsonString.length() - 1);
         }
         JsonFormat.parser().merge(functionDetailsJsonString, functionDetailsBuilder);
-        inferringMissingTypeClassName(functionDetailsBuilder, functionInstanceClassLoader);
+        FunctionCacheManager fnCache = new FunctionCacheManagerImpl(rootClassLoader);
+        ClassLoader functionClassLoader = ThreadRuntime.loadJars(jarFile, instanceConfig, functionId,
+                functionDetailsBuilder.getName(), narExtractionDirectory, fnCache);
+        inferringMissingTypeClassName(functionDetailsBuilder, functionClassLoader);
         Function.FunctionDetails functionDetails = functionDetailsBuilder.build();
         instanceConfig.setFunctionDetails(functionDetails);
         instanceConfig.setPort(port);
@@ -237,7 +243,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL
                         .tlsHostnameVerificationEnable(isTrue(tlsHostNameVerificationEnabled))
                         .tlsTrustCertsFilePath(tlsTrustCertFilePath).build(),
                 secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader,
-                exposePulsarAdminClientEnabled, webServiceUrl);
+                exposePulsarAdminClientEnabled, webServiceUrl, fnCache);
         runtimeSpawner = new RuntimeSpawner(
                 instanceConfig,
                 jarFile,
@@ -329,7 +335,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func
                     Map userConfigs = new Gson().fromJson(functionDetailsBuilder.getUserConfig(),
                             new TypeToken>() {
                             }.getType());
-                    boolean isWindowConfigPresent = userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY);
+                    boolean isWindowConfigPresent =
+                            userConfigs != null && userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY);
                     String className = functionDetailsBuilder.getClassName();
                     if (isWindowConfigPresent) {
                         WindowConfig windowConfig = new Gson().fromJson(
@@ -360,7 +367,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func
             case SINK:
                 if ((functionDetailsBuilder.hasSink()
                         && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) {
-                    String typeArg = getSinkType(functionDetailsBuilder.getClassName(), classLoader).getName();
+                    String typeArg =
+                            getSinkType(functionDetailsBuilder.getSink().getClassName(), classLoader).getName();
 
                     Function.SinkSpec.Builder sinkBuilder =
                             Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink());
@@ -378,7 +386,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func
             case SOURCE:
                 if ((functionDetailsBuilder.hasSource()
                         && functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) {
-                    String typeArg = getSourceType(functionDetailsBuilder.getClassName(), classLoader).getName();
+                    String typeArg =
+                            getSourceType(functionDetailsBuilder.getSource().getClassName(), classLoader).getName();
 
                     Function.SourceSpec.Builder sourceBuilder =
                             Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource());
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
index 0aa0cd95aef09..ed128568bcf50 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
@@ -137,14 +137,16 @@ private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig,
                         .getClassLoader();
             }
         }
-        return loadJars(jarFile, instanceConfig, functionId, narExtractionDirectory, fnCache);
+        return loadJars(jarFile, instanceConfig, functionId, instanceConfig.getFunctionDetails().getName(),
+                narExtractionDirectory, fnCache);
     }
 
-    private static ClassLoader loadJars(String jarFile,
-                                 InstanceConfig instanceConfig,
-                                 String functionId,
-                                 String narExtractionDirectory,
-                                 FunctionCacheManager fnCache) throws Exception {
+    public static ClassLoader loadJars(String jarFile,
+                                       InstanceConfig instanceConfig,
+                                       String functionId,
+                                       String functionName,
+                                       String narExtractionDirectory,
+                                       FunctionCacheManager fnCache) throws Exception {
         if (jarFile == null) {
             return Thread.currentThread().getContextClassLoader();
         }
@@ -175,8 +177,9 @@ private static ClassLoader loadJars(String jarFile,
                     Collections.emptyList());
         }
 
-        log.info("Initialize function class loader for function {} at function cache manager, functionClassLoader: {}",
-                instanceConfig.getFunctionDetails().getName(), fnCache.getClassLoader(functionId));
+        log.info(
+                "Initialize function class loader for function {} at function cache manager, functionClassLoader: {}",
+                functionName, fnCache.getClassLoader(functionId));
 
         fnClassLoader = fnCache.getClassLoader(functionId);
         if (null == fnClassLoader) {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
index 7bc055b25d6b9..cb9ad27a2dff8 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
@@ -86,7 +86,21 @@ public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl,
                 stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
                 narExtractionDirectory,
                 rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
-                Optional.empty());
+                Optional.empty(), null);
+    }
+
+    public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl,
+                                String stateStorageImplClass,
+                                String storageServiceUrl,
+                                AuthenticationConfig authConfig, SecretsProvider secretsProvider,
+                                FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory,
+                                ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled,
+                                String pulsarWebServiceUrl, FunctionCacheManager fnCache) throws Exception {
+        initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig,
+                stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
+                narExtractionDirectory,
+                rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
+                Optional.empty(), fnCache);
     }
 
     private void initialize(String threadGroupName, Optional memoryLimit,
@@ -96,7 +110,7 @@ private void initialize(String threadGroupName, Optional connectorsManager,
-                            Optional functionsManager)
+                            Optional functionsManager, FunctionCacheManager fnCache)
             throws PulsarClientException {
 
         if (rootClassLoader == null) {
@@ -106,7 +120,10 @@ private void initialize(String threadGroupName, Optional
Date: Wed, 24 May 2023 12:20:42 +0200
Subject: [PATCH 314/404] [improve][bk] Add integration test with bookie http
 server enabled (#20149)

Signed-off-by: tison 
Co-authored-by: tison 
---
 ...eeperInstallWithHttpServerEnabledTest.java | 84 +++++++++++++++++++
 .../integration/topologies/PulsarCluster.java | 36 +++++---
 .../topologies/PulsarClusterSpec.java         | 10 +++
 .../topologies/PulsarClusterTestBase.java     |  2 +
 4 files changed, 120 insertions(+), 12 deletions(-)
 create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java

diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java
new file mode 100644
index 0000000000000..03d7f974ab39b
--- /dev/null
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.integration.bookkeeper;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.tests.integration.docker.ContainerExecResult;
+import org.apache.pulsar.tests.integration.topologies.PulsarCluster;
+import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec;
+import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.joining;
+import static org.testng.Assert.assertEquals;
+
+/**
+ * Test bookkeeper setup with http server enabled.
+ */
+@Slf4j
+public class BookkeeperInstallWithHttpServerEnabledTest extends PulsarClusterTestBase {
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    public final void setupCluster() throws Exception {
+        incrementSetupNumber();
+
+        final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5))
+                .filter(s -> !s.isEmpty())
+                .collect(joining("-"));
+        bookkeeperEnvs.put("httpServerEnabled", "true");
+        bookieAdditionalPorts.add(8000);
+        PulsarClusterSpec spec = PulsarClusterSpec.builder()
+                .numBookies(2)
+                .numBrokers(1)
+                .bookkeeperEnvs(bookkeeperEnvs)
+                .bookieAdditionalPorts(bookieAdditionalPorts)
+                .clusterName(clusterName)
+                .build();
+
+        log.info("Setting up cluster {} with {} bookies, {} brokers",
+                spec.clusterName(), spec.numBookies(), spec.numBrokers());
+
+        pulsarCluster = PulsarCluster.forSpec(spec);
+        pulsarCluster.start();
+
+        log.info("Cluster {} is setup", spec.clusterName());
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    public final void tearDownCluster() throws Exception {
+        super.tearDownCluster();
+    }
+
+    @Test
+    public void testBookieHttpServerIsRunning() throws Exception {
+        ContainerExecResult result = pulsarCluster.getAnyBookie().execCmd(
+                PulsarCluster.CURL,
+                "-X",
+                "GET",
+                "http://localhost:8000/heartbeat");
+        assertEquals(result.getExitCode(), 0);
+        assertEquals(result.getStdout(), "OK\n");
+    }
+}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
index fcc0feec6d44f..bd11b7d387383 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
@@ -157,18 +157,26 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s
 
         // create bookies
         bookieContainers.putAll(
-                runNumContainers("bookie", spec.numBookies(), (name) -> new BKContainer(clusterName, name)
-                        .withNetwork(network)
-                        .withNetworkAliases(appendClusterName(name))
-                        .withEnv("zkServers", appendClusterName(ZKContainer.NAME))
-                        .withEnv("useHostNameAsBookieID", "true")
-                        // Disable fsyncs for tests since they're slow within the containers
-                        .withEnv("journalSyncData", "false")
-                        .withEnv("journalMaxGroupWaitMSec", "0")
-                        .withEnv("clusterName", clusterName)
-                        .withEnv("diskUsageThreshold", "0.99")
-                        .withEnv("nettyMaxFrameSizeBytes", "" + spec.maxMessageSize)
-                )
+                runNumContainers("bookie", spec.numBookies(), (name) -> {
+                    BKContainer bookieContainer = new BKContainer(clusterName, name)
+                            .withNetwork(network)
+                            .withNetworkAliases(appendClusterName(name))
+                            .withEnv("zkServers", appendClusterName(ZKContainer.NAME))
+                            .withEnv("useHostNameAsBookieID", "true")
+                            // Disable fsyncs for tests since they're slow within the containers
+                            .withEnv("journalSyncData", "false")
+                            .withEnv("journalMaxGroupWaitMSec", "0")
+                            .withEnv("clusterName", clusterName)
+                            .withEnv("diskUsageThreshold", "0.99")
+                            .withEnv("nettyMaxFrameSizeBytes", String.valueOf(spec.maxMessageSize));
+                    if (spec.bookkeeperEnvs != null) {
+                        bookieContainer.withEnv(spec.bookkeeperEnvs);
+                    }
+                    if (spec.bookieAdditionalPorts != null) {
+                        spec.bookieAdditionalPorts.forEach(bookieContainer::addExposedPort);
+                    }
+                    return bookieContainer;
+                })
         );
 
         // create brokers
@@ -740,4 +748,8 @@ public void dumpFunctionLogs(String name) {
     private String appendClusterName(String name) {
         return sharedCsContainer ? clusterName + "-" + name : name;
     }
+
+    public BKContainer getAnyBookie() {
+        return getAnyContainer(bookieContainers, "bookie");
+    }
 }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java
index 385af99a6644b..fa28d20e6b356 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java
@@ -150,6 +150,11 @@ public class PulsarClusterSpec {
      */
     Map brokerEnvs;
 
+    /**
+     * Specify envs for bookkeeper.
+     */
+    Map bookkeeperEnvs;
+
     /**
      * Specify mount files.
      */
@@ -167,4 +172,9 @@ public class PulsarClusterSpec {
      * Additional ports to expose on broker containers.
      */
     List brokerAdditionalPorts;
+
+    /**
+     * Additional ports to expose on bookie containers.
+     */
+    List bookieAdditionalPorts;
 }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java
index d7a1906ec582e..ae9e44fa98254 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java
@@ -34,8 +34,10 @@
 @Slf4j
 public abstract class PulsarClusterTestBase extends PulsarTestBase {
     protected final Map brokerEnvs = new HashMap<>();
+    protected final Map bookkeeperEnvs = new HashMap<>();
     protected final Map proxyEnvs = new HashMap<>();
     protected final List brokerAdditionalPorts = new LinkedList<>();
+    protected final List bookieAdditionalPorts = new LinkedList<>();
 
     @Override
     protected final void setup() throws Exception {

From 7dcb3eab2916d42bb58fc1639dc58e11b5230997 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= 
Date: Wed, 24 May 2023 21:56:30 +0800
Subject: [PATCH 315/404] [fix][broker] Invalidate metadata children cache
 after key deleted (#20363)

---
 .../org/apache/pulsar/metadata/impl/AbstractMetadataStore.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
index 072d513cca962..6fcf8eb6b49b2 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
@@ -328,6 +328,7 @@ public void accept(Notification n) {
 
         if (type == NotificationType.Created || type == NotificationType.Deleted) {
             existsCache.synchronous().invalidate(path);
+            childrenCache.synchronous().invalidate(path);
             String parent = parent(path);
             if (parent != null) {
                 childrenCache.synchronous().invalidate(parent);
@@ -385,6 +386,7 @@ private CompletableFuture deleteInternal(String path, Optional expec
         // Ensure caches are invalidated before the operation is confirmed
         return storeDelete(path, expectedVersion).thenRun(() -> {
             existsCache.synchronous().invalidate(path);
+            childrenCache.synchronous().invalidate(path);
             String parent = parent(path);
             if (parent != null) {
                 childrenCache.synchronous().invalidate(parent);

From fd36fc1f0d9c12201e5375982ced9ee4b61af944 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 24 May 2023 17:29:05 +0300
Subject: [PATCH 316/404] [improve][ci] Split Pulsar IO unit test job to
 multiple jobs (#20384)

---
 .github/workflows/pulsar-ci.yaml |  4 ++++
 build/run_unit_group.sh          | 12 ++++++++++++
 pom.xml                          | 14 ++++++++++++++
 pulsar-io/pom.xml                | 27 +++++++++++++++++++++------
 4 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index b92599581cd83..2c4abdb704178 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -198,6 +198,10 @@ jobs:
           - name: Pulsar IO
             group: PULSAR_IO
             timeout: 75
+          - name: Pulsar IO - Elastic Search
+            group: PULSAR_IO_ELASTIC
+          - name: Pulsar IO - Kafka Connect Adaptor
+            group: PULSAR_IO_KAFKA_CONNECT
           - name: Pulsar Client
             group: CLIENT
 
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index ba49820ed1d33..69434b011b37e 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -188,6 +188,18 @@ function test_group_pulsar_io() {
     echo "::endgroup::"
 }
 
+function test_group_pulsar_io_elastic() {
+    echo "::group::Running elastic-search tests"
+    mvn_test --install -Ppulsar-io-elastic-tests,-main
+    echo "::endgroup::"
+}
+
+function test_group_pulsar_io_kafka_connect() {
+    echo "::group::Running Pulsar IO Kafka connect adaptor tests"
+    mvn_test --install -Ppulsar-io-kafka-connect-tests,-main
+    echo "::endgroup::"
+}
+
 function list_test_groups() {
   declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]'
 }
diff --git a/pom.xml b/pom.xml
index 75c25e5478893..37349bbffadba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2438,6 +2438,20 @@ flexible messaging model and an intuitive client API.
       
     
 
+    
+      pulsar-io-elastic-tests
+      
+        pulsar-io
+      
+    
+
+    
+      pulsar-io-kafka-connect-tests
+      
+        pulsar-io
+      
+    
+
     
       pulsar-sql-tests
       
diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml
index 53079cdfbc807..c1a58d059cd48 100644
--- a/pulsar-io/pom.xml
+++ b/pulsar-io/pom.xml
@@ -85,22 +85,16 @@
         batch-discovery-triggerers
         batch-data-generator
         common
-        docs
         aws
         twitter
         cassandra
         aerospike
         http
-        kafka
         rabbitmq
         kinesis
         hdfs3
         jdbc
         data-generator
-        elastic-search
-        kafka-connect-adaptor
-        kafka-connect-adaptor-nar
-        debezium
         hdfs2
         canal
         file
@@ -117,6 +111,27 @@
       
     
 
+    
+      pulsar-io-elastic-tests
+      
+        core
+        common
+        elastic-search
+      
+    
+
+    
+      pulsar-io-kafka-connect-tests
+      
+        core
+        common
+        kafka
+        kafka-connect-adaptor
+        kafka-connect-adaptor-nar
+        debezium
+      
+    
+
     
       core-modules
       

From aff4a1c7edbf019d17795c7a67b1d0f0e701d118 Mon Sep 17 00:00:00 2001
From: vineeth1995 
Date: Wed, 24 May 2023 08:05:58 -0700
Subject: [PATCH 317/404] =?UTF-8?q?[fix][broker]=20Fix=20flaky=20test=20-?=
 =?UTF-8?q?=20testClusterMigrationWithReplica=E2=80=A6=20(#20379)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../apache/pulsar/broker/service/ClusterMigrationTest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
index df4f66c43d2b4..469e155d409b3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
@@ -458,8 +458,8 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc
         retryStrategically((test) -> !topic1.isReplicationBacklogExist(), 10, 1000);
         assertFalse(topic1.isReplicationBacklogExist());
 
-        // verify that the producer1 is now is now connected to migrated cluster "r2" since backlog is cleared.
-        retryStrategically((test) -> topic2.getProducers().size()==2, 10, 500);
+        producer1.send("test".getBytes());
+        // verify that the producer1 is now connected to migrated cluster "r2" since backlog is cleared.
         assertEquals(topic2.getProducers().size(), 2);
     }
 

From 903425be3afbbcfeb0a7213b0b7d04afe5868151 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 24 May 2023 19:35:15 +0300
Subject: [PATCH 318/404] [fix][test] Fix flaky test
 NonPersistentTopicTest.testMsgDropStat (#20387)

---
 .../client/api/NonPersistentTopicTest.java    | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index 8527406496448..c41ab3e8ccc73 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -64,6 +64,7 @@
 import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.apache.pulsar.zookeeper.ZookeeperServerTest;
+import org.awaitility.Awaitility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
@@ -824,10 +825,12 @@ public void testMsgDropStat() throws Exception {
             stopBroker();
             startBroker();
             Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
-                    .receiverQueueSize(1).subscribe();
+                    .receiverQueueSize(1)
+                    .messageListener((c, msg) -> {}).subscribe();
 
             Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
-                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
+                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared)
+                    .messageListener((c, msg) -> {}).subscribe();
 
             ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName)
                 .enableBatching(false)
@@ -848,15 +851,19 @@ public void testMsgDropStat() throws Exception {
             }
             latch.await();
 
-            NonPersistentTopic topic = (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
-            pulsar.getBrokerService().updateRates();
-            NonPersistentTopicStats stats = topic.getStats(false, false, false);
-            NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
-            NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1");
-            NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2");
-            assertTrue(npStats.getMsgDropRate() > 0);
-            assertTrue(sub1Stats.getMsgDropRate() > 0);
-            assertTrue(sub2Stats.getMsgDropRate() > 0);
+            NonPersistentTopic topic =
+                    (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
+
+            Awaitility.await().untilAsserted(() -> {
+                pulsar.getBrokerService().updateRates();
+                NonPersistentTopicStats stats = topic.getStats(false, false, false);
+                NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
+                NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1");
+                NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2");
+                assertTrue(npStats.getMsgDropRate() > 0);
+                assertTrue(sub1Stats.getMsgDropRate() > 0);
+                assertTrue(sub2Stats.getMsgDropRate() > 0);
+            });
 
             producer.close();
             consumer.close();

From 545abfc428eab8dfb93314e1051c47d0f026fadf Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Thu, 25 May 2023 00:36:39 +0800
Subject: [PATCH 319/404] [improve][doc] Improve doc for delayed message
 (#20374)

---
 conf/broker.conf                              |  8 +++-----
 conf/standalone.conf                          |  6 ++----
 .../pulsar/broker/ServiceConfiguration.java   | 19 +++++++++----------
 .../data/stats/SubscriptionStatsImpl.java     |  2 +-
 .../policies/data/stats/TopicStatsImpl.java   |  2 +-
 5 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 1183049bf8531..ca118d254f496 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -550,13 +550,11 @@ delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemory
 
 # Control the tick time for when retrying on delayed delivery,
 # affecting the accuracy of the delivery time compared to the scheduled time.
-# Note that this time is used to configure the HashedWheelTimer's tick time for the
-# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory).
+# Note that this time is used to configure the HashedWheelTimer's tick time.
 # Default is 1 second.
 delayedDeliveryTickTimeMillis=1000
 
-# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether
-# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
+# Whether the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
 # time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index
 # for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time
 # has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the
@@ -582,7 +580,7 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
 delayedDeliveryMaxNumBuckets=-1
 
 # Size of the lookahead window to use when detecting if all the messages in the topic
-# have a fixed delay.
+# have a fixed delay for InMemoryDelayedDeliveryTracker (the default DelayedDeliverTracker).
 # Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle
 # fixed delays in messages in a different way.
 delayedDeliveryFixedDelayDetectionLookahead=50000
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 19521bb74696b..8d24d5ad88c7f 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -1237,13 +1237,11 @@ delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemory
 
 # Control the tick time for when retrying on delayed delivery,
 # affecting the accuracy of the delivery time compared to the scheduled time.
-# Note that this time is used to configure the HashedWheelTimer's tick time for the
-# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory).
+# Note that this time is used to configure the HashedWheelTimer's tick time.
 # Default is 1 second.
 delayedDeliveryTickTimeMillis=1000
 
-# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether
-# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
+# Whether the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
 # time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index
 # for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time
 # has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 9966912bc8eae..6fee0e7cd09d2 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -343,17 +343,15 @@ public class ServiceConfiguration implements PulsarConfiguration {
 
     @FieldContext(category = CATEGORY_SERVER, doc = "Control the tick time for when retrying on delayed delivery, "
             + "affecting the accuracy of the delivery time compared to the scheduled time. Default is 1 second. "
-            + "Note that this time is used to configure the HashedWheelTimer's tick time for the "
-            + "InMemoryDelayedDeliveryTrackerFactory.")
+            + "Note that this time is used to configure the HashedWheelTimer's tick time.")
     private long delayedDeliveryTickTimeMillis = 1000;
 
-    @FieldContext(category = CATEGORY_SERVER, doc = "When using the InMemoryDelayedDeliveryTrackerFactory (the default "
-            + "DelayedDeliverTrackerFactory), whether the deliverAt time is strictly followed. When false (default), "
-            + "messages may be sent to consumers before the deliverAt time by as much as the tickTimeMillis. This can "
-            + "reduce the overhead on the broker of maintaining the delayed index for a potentially very short time "
-            + "period. When true, messages will not be sent to consumer until the deliverAt time has passed, and they "
-            + "may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the "
-            + "delayedDeliveryTickTimeMillis.")
+    @FieldContext(category = CATEGORY_SERVER, doc = "Whether the deliverAt time is strictly followed. "
+            + "When false (default), messages may be sent to consumers before the deliverAt time by as much "
+            + "as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index "
+            + "for a potentially very short time period. When true, messages will not be sent to consumer until the "
+            + "deliverAt time has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for "
+            + "the topic plus the delayedDeliveryTickTimeMillis.")
     private boolean isDelayedDeliveryDeliverAtTimeStrict = false;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
@@ -379,7 +377,8 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     private int delayedDeliveryMaxNumBuckets = -1;
 
     @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use "
-            + "when detecting if all the messages in the topic have a fixed delay. "
+            + "when detecting if all the messages in the topic have a fixed delay for "
+            + "InMemoryDelayedDeliveryTracker (the default DelayedDeliverTracker). "
             + "Default is 50,000. Setting the lookahead window to 0 will disable the "
             + "logic to handle fixed delays in messages in a different way.")
     private long delayedDeliveryFixedDelayDetectionLookahead = 50_000;
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index ea7639a8cd2c6..2bfa54da6a002 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -131,7 +131,7 @@ public class SubscriptionStatsImpl implements SubscriptionStats {
     /** The serialized size of non-contiguous deleted messages ranges. */
     public int nonContiguousDeletedMessagesRangesSerializedSize;
 
-    /** The size of InMemoryDelayedDeliveryTracer memory usage. */
+    /** The size of DelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
     public Map bucketDelayedIndexStats;
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index c9c4739b904f6..f0141bd58d38f 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -136,7 +136,7 @@ public class TopicStatsImpl implements TopicStats {
     /** The serialized size of non-contiguous deleted messages ranges. */
     public int nonContiguousDeletedMessagesRangesSerializedSize;
 
-    /** The size of InMemoryDelayedDeliveryTracer memory usage. */
+    /** The size of DelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
     /** Map of bucket delayed index statistics. */

From 9918bced4465e0b0746a7959550c90cb76ae945f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 25 May 2023 03:46:31 -0500
Subject: [PATCH 320/404] [fix][broker] partitioned __change_events topic is
 policy topic (#20392)

---
 .../service/persistent/PersistentTopic.java   |  2 +-
 .../common/naming/SystemTopicNames.java       |  6 +--
 .../common/naming/SystemTopicNamesTest.java   | 47 +++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 15854f55c5cd1..98e51a2e3ed6e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -819,7 +819,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St
             }
 
             try {
-                if (!topic.endsWith(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME)
+                if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic)
                         && !checkSubscriptionTypesEnable(subType)) {
                     return FutureUtil.failedFuture(
                             new NotAllowedException("Topic[{" + topic + "}] doesn't support "
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java
index 8fc7d014b5784..716d9bc31facb 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java
@@ -81,7 +81,7 @@ public static boolean isTopicPoliciesSystemTopic(String topic) {
         if (topic == null) {
             return false;
         }
-        return TopicName.get(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME);
+        return TopicName.getPartitionedTopicName(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME);
     }
 
     public static boolean isTransactionInternalName(TopicName topicName) {
@@ -92,7 +92,7 @@ public static boolean isTransactionInternalName(TopicName topicName) {
     }
 
     public static boolean isSystemTopic(TopicName topicName) {
-        TopicName nonePartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName());
-        return isEventSystemTopic(nonePartitionedTopicName) || isTransactionInternalName(nonePartitionedTopicName);
+        TopicName nonPartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName());
+        return isEventSystemTopic(nonPartitionedTopicName) || isTransactionInternalName(nonPartitionedTopicName);
     }
 }
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java
new file mode 100644
index 0000000000000..92d93021973b1
--- /dev/null
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.naming;
+
+import static org.testng.AssertJUnit.assertEquals;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test
+public class SystemTopicNamesTest {
+
+    @DataProvider(name = "topicPoliciesSystemTopicNames")
+    public static Object[][] topicPoliciesSystemTopicNames() {
+        return new Object[][] {
+                {"persistent://public/default/__change_events", true},
+                {"persistent://public/default/__change_events-partition-0", true},
+                {"persistent://random-tenant/random-ns/__change_events", true},
+                {"persistent://random-tenant/random-ns/__change_events-partition-1", true},
+                {"persistent://public/default/not_really__change_events", false},
+                {"persistent://public/default/__change_events-diff-suffix", false},
+                {"persistent://a/b/not_really__change_events", false},
+        };
+    }
+
+    @Test(dataProvider = "topicPoliciesSystemTopicNames")
+    public void testIsTopicPoliciesSystemTopic(String topicName, boolean expectedResult) {
+        assertEquals(expectedResult, SystemTopicNames.isTopicPoliciesSystemTopic(topicName));
+        assertEquals(expectedResult, SystemTopicNames.isSystemTopic(TopicName.get(topicName)));
+        assertEquals(expectedResult, SystemTopicNames.isEventSystemTopic(TopicName.get(topicName)));
+    }
+}

From 25c4b7cee402a1d486d720a71dc2e06aa6d9af64 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 25 May 2023 19:28:18 +0300
Subject: [PATCH 321/404] [fix][test] Fix flaky test
 NonPersistentTopicTest.testMsgDropStat, follow up (#20401)

Fixes #20386

### Motivation

- the previous attempt #20387 to fix the flakiness wasn't effective

### Modifications

- improve the fix and rely on the fact that entryId is -1 when the message is dropped in the broker
  code: https://github.com/apache/pulsar/blob/091ee2504ffbe6ec98e354b76e7f4c045e1914aa/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java#L1699-L1711

### Documentation



- [ ] `doc` 
- [ ] `doc-required` 
- [x] `doc-not-needed` 
- [ ] `doc-complete` 
---
 .../client/api/NonPersistentTopicTest.java    | 30 +++++++++++++------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index c41ab3e8ccc73..63ce0f00dff15 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -38,6 +38,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -50,6 +51,7 @@
 import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.impl.ConsumerImpl;
+import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl;
 import org.apache.pulsar.client.impl.PartitionedProducerImpl;
 import org.apache.pulsar.client.impl.ProducerImpl;
@@ -824,13 +826,14 @@ public void testMsgDropStat() throws Exception {
             conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
             stopBroker();
             startBroker();
+
+            pulsar.getBrokerService().updateRates();
+
             Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
-                    .receiverQueueSize(1)
-                    .messageListener((c, msg) -> {}).subscribe();
+                    .receiverQueueSize(1).subscribe();
 
             Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
-                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared)
-                    .messageListener((c, msg) -> {}).subscribe();
+                    .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
 
             ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName)
                 .enableBatching(false)
@@ -839,17 +842,26 @@ public void testMsgDropStat() throws Exception {
             @Cleanup("shutdownNow")
             ExecutorService executor = Executors.newFixedThreadPool(5);
             byte[] msgData = "testData".getBytes();
-            final int totalProduceMessages = 200;
-            CountDownLatch latch = new CountDownLatch(totalProduceMessages);
+            final int totalProduceMessages = 1000;
+            CountDownLatch latch = new CountDownLatch(1);
+            AtomicInteger messagesSent = new AtomicInteger(0);
             for (int i = 0; i < totalProduceMessages; i++) {
                 executor.submit(() -> {
-                    producer.sendAsync(msgData).handle((msg, e) -> {
-                        latch.countDown();
+                    producer.sendAsync(msgData).handle((msgId, e) -> {
+                        int count = messagesSent.incrementAndGet();
+                        // process at least 20% of messages before signalling the latch
+                        // a non-persistent message will return entryId as -1 when it has been dropped
+                        // due to setMaxConcurrentNonPersistentMessagePerConnection limit
+                        // also ensure that it has happened before the latch is signalled
+                        if (count > totalProduceMessages * 0.2 && msgId != null
+                                && ((MessageIdImpl) msgId).getEntryId() == -1) {
+                            latch.countDown();
+                        }
                         return null;
                     });
                 });
             }
-            latch.await();
+            latch.await(5, TimeUnit.SECONDS);
 
             NonPersistentTopic topic =
                     (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();

From f0e97f45e082854015dddea965ab9fc7b4d07e9b Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 25 May 2023 19:29:31 +0300
Subject: [PATCH 322/404] [improve][broker] Use immutable base cursor
 properties (#20400)

---
 .../service/persistent/PersistentSubscription.java       | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 09dabcd4bfc59..4e74857cef940 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -120,18 +120,15 @@ public class PersistentSubscription extends AbstractSubscription implements Subs
     // Map of properties that is used to mark this subscription as "replicated".
     // Since this is the only field at this point, we can just keep a static
     // instance of the map.
-    private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = new TreeMap<>();
-    private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Collections.emptyMap();
+    private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES =
+            Map.of(REPLICATED_SUBSCRIPTION_PROPERTY, 1L);
+    private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Map.of();
 
     private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache;
     private final PendingAckHandle pendingAckHandle;
     private volatile Map subscriptionProperties;
     private volatile CompletableFuture fenceFuture;
 
-    static {
-        REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L);
-    }
-
     static Map getBaseCursorProperties(boolean isReplicated) {
         return isReplicated ? REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES : NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES;
     }

From 795eb51762b417ea72bf201cdfe5c7585a8c3320 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 26 May 2023 18:25:52 +0300
Subject: [PATCH 323/404] [fix][ci] Update nar maven plugin version to fix
 excessive downloads (#20410)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 37349bbffadba..0fb86ad41a71d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -284,7 +284,7 @@ flexible messaging model and an intuitive client API.
     3.4.1
     3.1.0
     1.1.0
-    1.3.4
+    1.5.0
     3.1.2
     4.0.2
     3.5.3

From a953027aad38c9f54e952133949280ec2f4c04e8 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 26 May 2023 21:21:07 +0300
Subject: [PATCH 324/404] [fix][sec] Upgrade sqlite-jdbc to resolve
 CVE-2023-32697 (#20411)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 0fb86ad41a71d..b45debcc9db25 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,7 +181,7 @@ flexible messaging model and an intuitive client API.
     2.10.10
     2.5.0
     5.1.0
-    3.36.0.3
+    3.42.0.0
     8.0.11
     42.5.1
     0.4.6

From aa3bfcda6935dc2d9c69dce51f96ba80baeae0b7 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Sat, 27 May 2023 18:06:55 +0300
Subject: [PATCH 325/404] [improve][ci] Speed up OWASP dependency check in
 Pulsar CI workflow (#20412)

---
 .github/workflows/pulsar-ci.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 2c4abdb704178..57b9b082da266 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -1273,8 +1273,10 @@ jobs:
           cd $HOME
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
       # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
-      - name: run "clean verify" to trigger dependency check
-        run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
+      - name: trigger dependency check
+        run: |
+          mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
+            -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
 
       - name: Upload report
         uses: actions/upload-artifact@v3

From 1c813fdaeb9f752b84347b5a62e219dd66e79a26 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Mon, 29 May 2023 10:20:08 +0800
Subject: [PATCH 326/404] [fix][broker] Fix ledger cachemiss size metric
 (#20257)

---
 .../bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java       | 4 ++--
 .../bookkeeper/mledger/impl/cache/EntryCacheDisabled.java     | 4 ++--
 .../bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java    | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
index cb3d72cc5972f..e057dee99538e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
@@ -100,8 +100,8 @@ public void recordReadEntriesError() {
         readEntriesOpsFailed.recordEvent();
     }
 
-    public void recordReadEntriesOpsCacheMisses() {
-        readEntriesOpsCacheMisses.recordEvent();
+    public void recordReadEntriesOpsCacheMisses(int count, long totalSize) {
+        readEntriesOpsCacheMisses.recordMultipleEvents(count, totalSize);
     }
 
     public void addAddEntryLatencySample(long latency, TimeUnit unit) {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
index d2add99b701ac..d1050e0062826 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java
@@ -93,7 +93,7 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole
                     } finally {
                         ledgerEntries.close();
                     }
-                    ml.getMbean().recordReadEntriesOpsCacheMisses();
+                    ml.getMbean().recordReadEntriesOpsCacheMisses(entries.size(), totalSize);
                     ml.getFactory().getMbean().recordCacheMiss(entries.size(), totalSize);
                     ml.getMbean().addReadEntriesSample(entries.size(), totalSize);
 
@@ -121,7 +121,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks.
                             LedgerEntry ledgerEntry = iterator.next();
                             EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
 
-                            ml.getMbean().recordReadEntriesOpsCacheMisses();
+                            ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength());
                             ml.getFactory().getMbean().recordCacheMiss(1, returnEntry.getLength());
                             ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
                             callback.readEntryComplete(returnEntry, ctx);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
index 7747f9bcd93b6..27aec6f178e39 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
@@ -256,7 +256,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt
                                 LedgerEntry ledgerEntry = iterator.next();
                                 EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
 
-                                ml.getMbean().recordReadEntriesOpsCacheMisses();
+                                ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength());
                                 manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength());
                                 ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
                                 callback.readEntryComplete(returnEntry, ctx);
@@ -450,7 +450,7 @@ CompletableFuture> readFromStorage(ReadHandle lh,
                                     }
                                 }
 
-                                ml.getMbean().recordReadEntriesOpsCacheMisses();
+                                ml.getMbean().recordReadEntriesOpsCacheMisses(entriesToReturn.size(), totalSize);
                                 manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize);
                                 ml.getMbean().addReadEntriesSample(entriesToReturn.size(), totalSize);
 

From ab810f4f59dd7d6c8b5313ceb334873a7d2cde31 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Mon, 29 May 2023 14:46:38 +0800
Subject: [PATCH 327/404] [fix][broker] If ledger lost, cursor mark delete
 position can not forward (#18620)

Motivation:
Configuration `autoSkipNonRecoverableData` is designed to turn this feature on if we can accept partial data loss. When a ledger is lost, the broker will still work. But now we have this problem: If a ledger is lost, consumer and producer can work, but the cursor mark delete position can not forward.

Modifications:
- When an unrecoverable ledger is found, remove the records in`individualDeletedMessages` and `batchDeletedIndexes`.
- When the managed cursor is recovered, check whether there are invalid records in `individualDeletedMessages` and `batchDeletedIndexes` and print a warning log.
---
 .../bookkeeper/mledger/ManagedCursor.java     |   6 +
 .../bookkeeper/mledger/ManagedLedger.java     |   6 +
 .../mledger/impl/ManagedCursorImpl.java       |  40 +++
 .../mledger/impl/ManagedLedgerImpl.java       |   7 +
 .../bookkeeper/mledger/impl/OpReadEntry.java  |   5 +
 .../LedgerLostAndSkipNonRecoverableTest.java  | 296 ++++++++++++++++++
 6 files changed, 360 insertions(+)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index 7802ed07781ba..edbfa0b43204e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -786,6 +786,12 @@ Set asyncReplayEntries(
      */
     long getEstimatedSizeSinceMarkDeletePosition();
 
+    /**
+     * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is
+     * used to delete information about this ledger in the ManagedCursor.
+     */
+    default void skipNonRecoverableLedger(long ledgerId){}
+
     /**
      * Returns cursor throttle mark-delete rate.
      *
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
index 4ca56508891a1..c7dd8ea9129b7 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
@@ -631,6 +631,12 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro
      */
     void trimConsumedLedgersInBackground(CompletableFuture promise);
 
+    /**
+     * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is
+     * used to delete information about this ledger in the ManagedCursor.
+     */
+    default void skipNonRecoverableLedger(long ledgerId){}
+
     /**
      * Roll current ledger if it is full.
      */
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index ef607fa7ed7cf..663081c932052 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -2718,6 +2718,46 @@ void setReadPosition(Position newReadPositionInt) {
         }
     }
 
+    /**
+     * Manually acknowledge all entries in the lost ledger.
+     * - Since this is an uncommon event, we focus on maintainability. So we do not modify
+     *   {@link #individualDeletedMessages} and {@link #batchDeletedIndexes}, but call
+     *   {@link #asyncDelete(Position, AsyncCallbacks.DeleteCallback, Object)}.
+     * - This method is valid regardless of the consumer ACK type.
+     * - If there is a consumer ack request after this event, it will also work.
+     */
+    @Override
+    public void skipNonRecoverableLedger(final long ledgerId){
+        LedgerInfo ledgerInfo = ledger.getLedgersInfo().get(ledgerId);
+        if (ledgerInfo == null) {
+            return;
+        }
+        lock.writeLock().lock();
+        log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will"
+                + " be auto acknowledge in subscription", ledger.getName(), name, ledgerId);
+        try {
+            for (int i = 0; i < ledgerInfo.getEntries(); i++) {
+                if (!individualDeletedMessages.contains(ledgerId, i)) {
+                    asyncDelete(PositionImpl.get(ledgerId, i), new AsyncCallbacks.DeleteCallback() {
+                        @Override
+                        public void deleteComplete(Object ctx) {
+                            // ignore.
+                        }
+
+                        @Override
+                        public void deleteFailed(ManagedLedgerException ex, Object ctx) {
+                            // The method internalMarkDelete already handled the failure operation. We only need to
+                            // make sure the memory state is updated.
+                            // If the broker crashed, the non-recoverable ledger will be detected again.
+                        }
+                    }, null);
+                }
+            }
+        } finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     // //////////////////////////////////////////////////
 
     void startCreatingNewMetadataLedger() {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 15e9d332fa103..9b3d7e46aaa8c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1741,6 +1741,13 @@ synchronized void ledgerClosed(final LedgerHandle lh) {
         }
     }
 
+    @Override
+    public void skipNonRecoverableLedger(long ledgerId){
+        for (ManagedCursor managedCursor : cursors) {
+            managedCursor.skipNonRecoverableLedger(ledgerId);
+        }
+    }
+
     synchronized void createLedgerAfterClosed() {
         if (isNeededCreateNewLedgerAfterCloseLedger()) {
             log.info("[{}] Creating a new ledger after closed", name);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
index 19211553a5f74..7b59c3903d5bc 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
@@ -116,9 +116,11 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
                     readPosition, exception.getMessage());
             final ManagedLedgerImpl ledger = (ManagedLedgerImpl) cursor.getManagedLedger();
             Position nexReadPosition;
+            Long lostLedger = null;
             if (exception instanceof ManagedLedgerException.LedgerNotExistException) {
                 // try to find and move to next valid ledger
                 nexReadPosition = cursor.getNextLedgerPosition(readPosition.getLedgerId());
+                lostLedger = readPosition.ledgerId;
             } else {
                 // Skip this read operation
                 nexReadPosition = ledger.getValidPositionAfterSkippedEntries(readPosition, count);
@@ -131,6 +133,9 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
                 return;
             }
             updateReadPosition(nexReadPosition);
+            if (lostLedger != null) {
+                cursor.getManagedLedger().skipNonRecoverableLedger(lostLedger);
+            }
             checkReadCompletion();
         } else {
             if (!(exception instanceof TooManyRequestsException)) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
new file mode 100644
index 0000000000000..389af8f2cd9f9
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.BatchMessageIdImpl;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class LedgerLostAndSkipNonRecoverableTest extends ProducerConsumerBase {
+
+    private static final String DEFAULT_NAMESPACE = "my-property/my-ns";
+
+    @BeforeClass
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+    }
+
+    @AfterClass
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    protected void doInitConf() throws Exception {
+        conf.setAutoSkipNonRecoverableData(true);
+    }
+
+    @DataProvider(name = "batchEnabled")
+    public Object[][] batchEnabled(){
+        return new Object[][]{
+                {true},
+                {false}
+        };
+    }
+
+    @Test(timeOut = 30000, dataProvider = "batchEnabled")
+    public void testMarkDeletedPositionCanForwardAfterTopicLedgerLost(boolean enabledBatch) throws Exception {
+        String topicSimpleName = UUID.randomUUID().toString().replaceAll("-", "");
+        String subName = UUID.randomUUID().toString().replaceAll("-", "");
+        String topicName = String.format("persistent://%s/%s", DEFAULT_NAMESPACE, topicSimpleName);
+
+        log.info("create topic and subscription.");
+        Consumer sub = createConsumer(topicName, subName, enabledBatch);
+        sub.redeliverUnacknowledgedMessages();
+        sub.close();
+
+        log.info("send many messages.");
+        int ledgerCount = 3;
+        int messageCountPerLedger = enabledBatch ? 25 : 5;
+        int messageCountPerEntry = enabledBatch ? 5 : 1;
+        List[] sendMessages =
+                sendManyMessages(topicName, ledgerCount, messageCountPerLedger, messageCountPerEntry);
+        int sendMessageCount = Arrays.asList(sendMessages).stream()
+                .flatMap(s -> s.stream()).collect(Collectors.toList()).size();
+        log.info("send {} messages", sendMessageCount);
+
+        log.info("make individual ack.");
+        ConsumerAndReceivedMessages consumerAndReceivedMessages1 =
+                waitConsumeAndAllMessages(topicName, subName, enabledBatch,false);
+        List[] messageIds = consumerAndReceivedMessages1.messageIds;
+        Consumer consumer = consumerAndReceivedMessages1.consumer;
+        MessageIdImpl individualPosition = messageIds[1].get(messageCountPerEntry - 1);
+        MessageIdImpl expectedMarkDeletedPosition =
+                new MessageIdImpl(messageIds[0].get(0).getLedgerId(), messageIds[0].get(0).getEntryId(), -1);
+        MessageIdImpl lastPosition =
+                new MessageIdImpl(messageIds[2].get(4).getLedgerId(), messageIds[2].get(4).getEntryId(), -1);
+        consumer.acknowledge(individualPosition);
+        consumer.acknowledge(expectedMarkDeletedPosition);
+        waitPersistentCursorLedger(topicName, subName, expectedMarkDeletedPosition.getLedgerId(),
+                expectedMarkDeletedPosition.getEntryId());
+        consumer.close();
+
+        log.info("Make lost ledger [{}].", individualPosition.getLedgerId());
+        pulsar.getBrokerService().getTopic(topicName, false).get().get().close(false);
+        pulsarTestContext.getMockBookKeeper().deleteLedger(individualPosition.getLedgerId());
+
+        log.info("send some messages.");
+        sendManyMessages(topicName, 3, messageCountPerEntry);
+
+        log.info("receive all messages then verify mark deleted position");
+        ConsumerAndReceivedMessages consumerAndReceivedMessages2 =
+                waitConsumeAndAllMessages(topicName, subName, enabledBatch, true);
+        waitMarkDeleteLargeAndEquals(topicName, subName, lastPosition.getLedgerId(), lastPosition.getEntryId());
+
+        // cleanup
+        consumerAndReceivedMessages2.consumer.close();
+        admin.topics().delete(topicName);
+    }
+
+    private ManagedCursorImpl getCursor(String topicName, String subName) throws Exception {
+        PersistentSubscription subscription_ =
+                (PersistentSubscription) pulsar.getBrokerService().getTopic(topicName, false)
+                        .get().get().getSubscription(subName);
+        return  (ManagedCursorImpl) subscription_.getCursor();
+    }
+
+    private void waitMarkDeleteLargeAndEquals(String topicName, String subName, final long markDeletedLedgerId,
+                                            final long markDeletedEntryId) throws Exception {
+        Awaitility.await().atMost(Duration.ofSeconds(45)).untilAsserted(() -> {
+            Position persistentMarkDeletedPosition = getCursor(topicName, subName).getMarkDeletedPosition();
+            log.info("markDeletedPosition {}:{}, expected {}:{}", persistentMarkDeletedPosition.getLedgerId(),
+                    persistentMarkDeletedPosition.getEntryId(), markDeletedLedgerId, markDeletedEntryId);
+            Assert.assertTrue(persistentMarkDeletedPosition.getLedgerId() >= markDeletedLedgerId);
+            if (persistentMarkDeletedPosition.getLedgerId() > markDeletedLedgerId){
+                return;
+            }
+            Assert.assertTrue(persistentMarkDeletedPosition.getEntryId() >= markDeletedEntryId);
+        });
+    }
+
+    private void waitPersistentCursorLedger(String topicName, String subName, final long markDeletedLedgerId,
+                                            final long markDeletedEntryId) throws Exception {
+        Awaitility.await().untilAsserted(() -> {
+            Position persistentMarkDeletedPosition = getCursor(topicName, subName).getPersistentMarkDeletedPosition();
+            Assert.assertEquals(persistentMarkDeletedPosition.getLedgerId(), markDeletedLedgerId);
+            Assert.assertEquals(persistentMarkDeletedPosition.getEntryId(), markDeletedEntryId);
+        });
+    }
+
+    private List[] sendManyMessages(String topicName, int ledgerCount, int messageCountPerLedger,
+                                                   int messageCountPerEntry) throws Exception {
+        List[] messageIds = new List[ledgerCount];
+        for (int i = 0; i < ledgerCount; i++){
+            admin.topics().unload(topicName);
+            if (messageCountPerEntry == 1) {
+                messageIds[i] = sendManyMessages(topicName, messageCountPerLedger);
+            } else {
+                messageIds[i] = sendManyBatchedMessages(topicName, messageCountPerEntry,
+                        messageCountPerLedger / messageCountPerEntry);
+            }
+        }
+        return messageIds;
+    }
+
+    private List sendManyMessages(String topicName, int messageCountPerLedger,
+                                                   int messageCountPerEntry) throws Exception {
+        if (messageCountPerEntry == 1) {
+            return sendManyMessages(topicName, messageCountPerLedger);
+        } else {
+            return sendManyBatchedMessages(topicName, messageCountPerEntry,
+                    messageCountPerLedger / messageCountPerEntry);
+        }
+    }
+
+    private List sendManyMessages(String topicName, int msgCount) throws Exception {
+        List messageIdList = new ArrayList<>();
+        final Producer producer = pulsarClient.newProducer(Schema.JSON(String.class))
+                .topic(topicName)
+                .enableBatching(false)
+                .create();
+        long timestamp = System.currentTimeMillis();
+        for (int i = 0; i < msgCount; i++){
+            String messageSuffix = String.format("%s-%s", timestamp, i);
+            MessageIdImpl messageIdSent = (MessageIdImpl) producer.newMessage()
+                    .key(String.format("Key-%s", messageSuffix))
+                    .value(String.format("Msg-%s", messageSuffix))
+                    .send();
+            messageIdList.add(messageIdSent);
+        }
+        producer.close();
+        return messageIdList;
+    }
+
+    private List sendManyBatchedMessages(String topicName, int msgCountPerEntry, int entryCount)
+            throws Exception {
+        Producer producer = pulsarClient.newProducer(Schema.JSON(String.class))
+                .topic(topicName)
+                .enableBatching(true)
+                .batchingMaxPublishDelay(Integer.MAX_VALUE, TimeUnit.SECONDS)
+                .batchingMaxMessages(Integer.MAX_VALUE)
+                .create();
+        List> messageIdFutures = new ArrayList<>();
+        for (int i = 0; i < entryCount; i++){
+            for (int j = 0; j < msgCountPerEntry; j++){
+                CompletableFuture messageIdFuture =
+                        producer.newMessage().value(String.format("entry-seq[%s], batch_index[%s]", i, j)).sendAsync();
+                messageIdFutures.add(messageIdFuture);
+            }
+            producer.flush();
+        }
+        producer.close();
+        FutureUtil.waitForAll(messageIdFutures).get();
+        return messageIdFutures.stream().map(f -> (MessageIdImpl)f.join()).collect(Collectors.toList());
+    }
+
+    private ConsumerAndReceivedMessages waitConsumeAndAllMessages(String topicName, String subName,
+                                                            final boolean enabledBatch,
+                                                            boolean ack) throws Exception {
+        List messageIds = new ArrayList<>();
+        final Consumer consumer = createConsumer(topicName, subName, enabledBatch);
+        while (true){
+            Message message = consumer.receive(5, TimeUnit.SECONDS);
+            if (message != null){
+                messageIds.add((MessageIdImpl) message.getMessageId());
+                if (ack) {
+                    consumer.acknowledge(message);
+                }
+            } else {
+                break;
+            }
+        }
+        log.info("receive {} messages", messageIds.size());
+        return new ConsumerAndReceivedMessages(consumer, sortMessageId(messageIds, enabledBatch));
+    }
+
+    @AllArgsConstructor
+    private static class ConsumerAndReceivedMessages {
+        private Consumer consumer;
+        private List[] messageIds;
+    }
+
+    private List[] sortMessageId(List messageIds, boolean enabledBatch){
+        Map> map = messageIds.stream().collect(Collectors.groupingBy(v -> v.getLedgerId()));
+        TreeMap> sortedMap = new TreeMap<>(map);
+        List[] res = new List[sortedMap.size()];
+        Iterator>> iterator = sortedMap.entrySet().iterator();
+        for (int i = 0; i < sortedMap.size(); i++){
+            res[i] = iterator.next().getValue();
+        }
+        for (List list : res){
+            list.sort((m1, m2) -> {
+                if (enabledBatch){
+                    BatchMessageIdImpl mb1 = (BatchMessageIdImpl) m1;
+                    BatchMessageIdImpl mb2 = (BatchMessageIdImpl) m2;
+                    return (int) (mb1.getLedgerId() * 1000000 + mb1.getEntryId() * 1000 + mb1.getBatchIndex() -
+                            mb2.getLedgerId() * 1000000 + mb2.getEntryId() * 1000 + mb2.getBatchIndex());
+                }
+                return (int) (m1.getLedgerId() * 1000 + m1.getEntryId() -
+                        m2.getLedgerId() * 1000 + m2.getEntryId());
+            });
+        }
+        return res;
+    }
+
+    private Consumer createConsumer(String topicName, String subName, boolean enabledBatch) throws Exception {
+        final Consumer consumer = pulsarClient.newConsumer(Schema.JSON(String.class))
+                .autoScaledReceiverQueueSizeEnabled(false)
+                .subscriptionType(SubscriptionType.Failover)
+                .isAckReceiptEnabled(true)
+                .enableBatchIndexAcknowledgment(enabledBatch)
+                .receiverQueueSize(1000)
+                .topic(topicName)
+                .subscriptionName(subName)
+                .subscribe();
+        return consumer;
+    }
+}

From b5ef09e537ddb24419e9ad05b1dc940ab10ea70a Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Mon, 29 May 2023 18:01:24 +0800
Subject: [PATCH 328/404] [fix][test] Fix SegmentAbortedTxnProcessorTest
 (#20358)

---
 .../SegmentAbortedTxnProcessorTest.java       | 35 ++++++++++++++++---
 .../transaction/TransactionTestBase.java      |  6 ++--
 .../buffer/TransactionBufferCloseTest.java    |  4 ---
 3 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index cb15ab003f7b7..0600833b1adb6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -25,6 +25,7 @@
 import java.lang.reflect.Field;
 import java.util.LinkedList;
 import java.util.NavigableMap;
+import java.util.Optional;
 import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
@@ -38,10 +39,12 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
 import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
+import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl;
 import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes;
 import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment;
@@ -56,6 +59,7 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.TopicStats;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -71,11 +75,13 @@ public class SegmentAbortedTxnProcessorTest extends TransactionTestBase {
     @Override
     @BeforeClass
     protected void setup() throws Exception {
-        setUpBase(1, 1, PROCESSOR_TOPIC, 0);
+        setUpBase(1, 1, null, 0);
         this.pulsarService = getPulsarServiceList().get(0);
         this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
         this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() +
                 SEGMENT_SIZE * 3);
+        admin.topics().createNonPartitionedTopic(PROCESSOR_TOPIC);
+        assertTrue(getSnapshotAbortedTxnProcessor(PROCESSOR_TOPIC) instanceof SnapshotSegmentAbortedTxnProcessorImpl);
     }
 
     @Override
@@ -311,15 +317,18 @@ private void doCompaction(TopicName topic) throws Exception {
      */
     @Test
     public void testSnapshotProcessorUpgrade() throws Exception {
+        String NAMESPACE2 = TENANT + "/ns2";
+        admin.namespaces().createNamespace(NAMESPACE2);
         this.pulsarService = getPulsarServiceList().get(0);
         this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false);
 
         // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions.
         // Abort these transactions and verify the data.
-        final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade";
+        final String topicName = "persistent://" + NAMESPACE2 + "/testSnapshotProcessorUpgrade";
         Producer producer = pulsarClient.newProducer().topic(topicName).create();
         Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe();
 
+        assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SingleSnapshotAbortedTxnProcessorImpl);
         // Send 10 messages without using transactions
         for (int i = 0; i < 10; i++) {
             producer.send(("test-message-" + i).getBytes());
@@ -352,6 +361,7 @@ public void testSnapshotProcessorUpgrade() throws Exception {
 
         // Unload the topic
         admin.topics().unload(topicName);
+        assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl);
 
         // Sends a new message using a transaction and aborts it.
         Transaction txn = pulsarClient.newTransaction()
@@ -362,7 +372,7 @@ public void testSnapshotProcessorUpgrade() throws Exception {
 
         // Verifies that the topic has exactly one segment.
         Awaitility.await().untilAsserted(() -> {
-            String segmentTopic = "persistent://" + NAMESPACE1 + "/" +
+            String segmentTopic = "persistent://" + NAMESPACE2 + "/" +
                     SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS;
             TopicStats topicStats = admin.topics().getStats(segmentTopic);
             assertEquals(1, topicStats.getMsgInCounter());
@@ -401,7 +411,7 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except
         String topicName = "persistent://" + namespaceName + "/newTopic";
         Producer producer = pulsarClient.newProducer().topic(topicName).create();
         producer.close();
-
+        assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl);
         // Check that the __transaction_buffer_snapshot topic is not created in the same namespace
         String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" +
                 SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT;
@@ -415,4 +425,21 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except
         // Destroy the namespace after the test
         admin.namespaces().deleteNamespace(namespaceName, true);
     }
+
+    private AbortedTxnProcessor getSnapshotAbortedTxnProcessor(String topicName) {
+        PersistentTopic persistentTopic = getPersistentTopic(topicName);
+        return WhiteboxImpl.getInternalState(persistentTopic.getTransactionBuffer(), "snapshotAbortedTxnProcessor");
+    }
+
+    private PersistentTopic getPersistentTopic(String topicName) {
+        for (PulsarService pulsar : getPulsarServiceList()) {
+            CompletableFuture> future =
+                    pulsar.getBrokerService().getTopic(topicName, false);
+            if (future == null) {
+                continue;
+            }
+            return (PersistentTopic) future.join().get();
+        }
+        throw new NullPointerException("topic[" + topicName +  "] not found");
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
index f45eda8d21fbe..cd0c089ad41be 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
@@ -114,10 +114,10 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int
                 new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
         admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString());
         createTransactionCoordinatorAssign(numPartitionsOfTC);
+        admin.tenants().createTenant(TENANT,
+                new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
+        admin.namespaces().createNamespace(NAMESPACE1);
         if (topic != null) {
-            admin.tenants().createTenant(TENANT,
-                    new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
-            admin.namespaces().createNamespace(NAMESPACE1);
             if (numPartitions == 0) {
                 admin.topics().createNonPartitionedTopic(topic);
             } else {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
index e92cf29521e34..d1784f6a392bf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.broker.transaction.buffer;
 
-import com.google.common.collect.Sets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -30,7 +29,6 @@
 import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -49,8 +47,6 @@ protected void setup() throws Exception {
         setUpBase(1, 16, null, 0);
         Awaitility.await().until(() -> ((PulsarClientImpl) pulsarClient)
                 .getTcClient().getState() == TransactionCoordinatorClient.State.READY);
-        admin.tenants().createTenant(TENANT,
-                new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
     }
 
     @AfterMethod(alwaysRun = true)

From 6079a9186ac8d1c96c97828a289d7865c5890ebd Mon Sep 17 00:00:00 2001
From: Zike Yang 
Date: Mon, 29 May 2023 21:03:34 +0800
Subject: [PATCH 329/404] [fix][io] Close the kafka source connector if there
 is uncaught exception (#20424)

Fixes #19880

### Motivation

If any unexpected and uncaught exceptions are thrown in the `runnerThread` of the Kafka source connector, the connector will just log that message and get stuck. The connector won't exit but won't do anything either.

### Modifications

* Close the connector if there is uncaught exception

Signed-off-by: Zike Yang 
---
 .../pulsar/io/kafka/KafkaAbstractSource.java  |  9 +++++++-
 .../kafka/source/KafkaAbstractSourceTest.java | 23 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
index 565c36047474b..8d2cbd8e74e14 100644
--- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
+++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
@@ -189,7 +189,14 @@ public void start() {
             }
         });
         runnerThread.setUncaughtExceptionHandler(
-                (t, e) -> LOG.error("[{}] Error while consuming records", t.getName(), e));
+                (t, e) -> {
+                    LOG.error("[{}] Error while consuming records", t.getName(), e);
+                    try {
+                        this.close();
+                    } catch (InterruptedException ex) {
+                        // The interrupted exception is thrown by the runnerThread itself. Ignore it.
+                    }
+                });
         runnerThread.setName("Kafka Source Thread");
         runnerThread.start();
     }
diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java
index 612cf0bc6d2b1..bc06c3e1935b4 100644
--- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java
+++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java
@@ -20,7 +20,10 @@
 
 
 import com.google.common.collect.ImmutableMap;
+import java.util.Collection;
 import java.util.Collections;
+import java.lang.reflect.Field;
+import org.apache.kafka.clients.consumer.Consumer;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.security.auth.SecurityProtocol;
@@ -28,6 +31,7 @@
 import org.apache.pulsar.io.core.SourceContext;
 import org.apache.pulsar.io.kafka.KafkaAbstractSource;
 import org.apache.pulsar.io.kafka.KafkaSourceConfig;
+import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -153,6 +157,25 @@ public final void loadFromSaslYamlFileTest() throws IOException {
         assertEquals(config.getSslTruststorePassword(), "cert_pwd");
     }
 
+    @Test
+    public final void closeConnectorWhenUnexpectedExceptionThrownTest() throws Exception {
+        KafkaAbstractSource source = new DummySource();
+        Consumer consumer = mock(Consumer.class);
+        Mockito.doThrow(new RuntimeException("Uncaught exception")).when(consumer)
+                .subscribe(Mockito.any(Collection.class));
+
+        Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer");
+        consumerField.setAccessible(true);
+        consumerField.set(source, consumer);
+
+        source.start();
+
+        Field runningField = KafkaAbstractSource.class.getDeclaredField("running");
+        runningField.setAccessible(true);
+
+        Assert.assertFalse((boolean) runningField.get(source));
+    }
+
     private File getFile(String name) {
         ClassLoader classLoader = getClass().getClassLoader();
         return new File(classLoader.getResource(name).getFile());

From 0bdcaecd38edbf4597261240aa47aa50c879921e Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 29 May 2023 16:33:46 +0300
Subject: [PATCH 330/404] [improve][ci] Improve TestNG test execution to debug
 flaky test failures (#20425)

---
 .../apache/pulsar/tests/FailFastNotifier.java |  3 +--
 .../pulsar/tests/PulsarTestListener.java      | 25 +++++++++++++------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java
index 627a4ec30547b..fe76a79b2c4ce 100644
--- a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java
@@ -124,8 +124,7 @@ public void beforeInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestRes
                         || iTestNGMethod.isAfterTestConfiguration())) {
                     throw new FailFastSkipException("Skipped after failure since testFailFast system property is set.");
                 }
-            }
-            if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) {
+            } else if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) {
                 throw new FailFastSkipException("Skipped after failure since kill switch file exists.");
             }
         }
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java
index b3d70621843ca..2d1f1273272c5 100644
--- a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java
@@ -44,20 +44,29 @@ public void onTestFailure(ITestResult result) {
         if (!(result.getThrowable() instanceof SkipException)) {
             System.out.format("!!!!!!!!! FAILURE-- %s.%s(%s)-------\n", result.getTestClass(),
                     result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
-        }
-        if (result.getThrowable() != null) {
-            result.getThrowable().printStackTrace();
-            if (result.getThrowable() instanceof ThreadTimeoutException) {
-                System.out.println("====== THREAD DUMPS ======");
-                System.out.println(ThreadDumpUtil.buildThreadDiagnosticString());
+            if (result.getThrowable() != null) {
+                result.getThrowable().printStackTrace();
+                if (result.getThrowable() instanceof ThreadTimeoutException) {
+                    System.out.println("====== THREAD DUMPS ======");
+                    System.out.println(ThreadDumpUtil.buildThreadDiagnosticString());
+                }
             }
         }
     }
 
     @Override
     public void onTestSkipped(ITestResult result) {
-        System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(),
-                result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
+        if (!(result.getThrowable() instanceof SkipException)) {
+            System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(),
+                    result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
+            if (result.getThrowable() != null) {
+                result.getThrowable().printStackTrace();
+                if (result.getThrowable() instanceof ThreadTimeoutException) {
+                    System.out.println("====== THREAD DUMPS ======");
+                    System.out.println(ThreadDumpUtil.buildThreadDiagnosticString());
+                }
+            }
+        }
     }
 
     @Override

From fafadee9061f1256cbb73385415bb9b978e74401 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Mon, 29 May 2023 22:11:28 +0800
Subject: [PATCH 331/404] [improve] [broker] Avoid
 `PersistentSubscription.expireMessages` logic check backlog twice. (#20416)

---
 .../broker/service/persistent/PersistentSubscription.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 4e74857cef940..dc666f3a18e48 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -1048,8 +1048,9 @@ public List getConsumers() {
 
     @Override
     public boolean expireMessages(int messageTTLInSeconds) {
-        if ((getNumberOfEntriesInBacklog(false) == 0) || (dispatcher != null && dispatcher.isConsumerConnected()
-                && getNumberOfEntriesInBacklog(false) < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK
+        long backlog = getNumberOfEntriesInBacklog(false);
+        if (backlog == 0 || (dispatcher != null && dispatcher.isConsumerConnected()
+                && backlog < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK
                 && !topic.isOldestMessageExpired(cursor, messageTTLInSeconds))) {
             // don't do anything for almost caught-up connected subscriptions
             return false;

From bdd1bf1fcabdfc52527d07a8f8a288930e2cb7dd Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 29 May 2023 20:22:05 +0300
Subject: [PATCH 332/404] [improve][broker] Replace String.intern() with Guava
 Interner (#20432)

---
 .../broker/service/AbstractReplicator.java    |  7 +--
 .../pulsar/broker/service/ServerCnx.java      |  3 +-
 .../pulsar/common/util/StringInterner.java    | 46 +++++++++++++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
index deab89cda72dc..d38a5da3adba5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.client.impl.PulsarClientImpl;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.util.StringInterner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,9 +74,9 @@ public AbstractReplicator(String localCluster, String localTopicName, String rem
         this.brokerService = brokerService;
         this.localTopicName = localTopicName;
         this.replicatorPrefix = replicatorPrefix;
-        this.localCluster = localCluster.intern();
+        this.localCluster = StringInterner.intern(localCluster);
         this.remoteTopicName = remoteTopicName;
-        this.remoteCluster = remoteCluster.intern();
+        this.remoteCluster = StringInterner.intern(remoteCluster);
         this.replicationClient = replicationClient;
         this.client = (PulsarClientImpl) brokerService.pulsar().getClient();
         this.producer = null;
@@ -228,7 +229,7 @@ public static String getRemoteCluster(String remoteCursor) {
     }
 
     public static String getReplicatorName(String replicatorPrefix, String cluster) {
-        return (replicatorPrefix + "." + cluster).intern();
+        return StringInterner.intern(replicatorPrefix + "." + cluster);
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 888668e15b167..d3c3971dfaced 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -164,6 +164,7 @@
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.util.StringInterner;
 import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
 import org.apache.pulsar.common.util.netty.NettyChannelUtil;
 import org.apache.pulsar.common.util.netty.NettyFutureUtil;
@@ -717,7 +718,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) {
         }
         setRemoteEndpointProtocolVersion(clientProtoVersion);
         if (isNotBlank(clientVersion)) {
-            this.clientVersion = clientVersion.intern();
+            this.clientVersion = StringInterner.intern(clientVersion);
         }
         if (!service.isAuthenticationEnabled()) {
             log.info("[{}] connected with clientVersion={}, clientProtocolVersion={}, proxyVersion={}", remoteAddress,
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java
new file mode 100644
index 0000000000000..3f6b1c453cdbc
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.util;
+
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+
+/**
+ * Deduplicates String instances by interning them using Guava's Interner
+ * which is more efficient than String.intern().
+ */
+public class StringInterner {
+    private static final StringInterner INSTANCE = new StringInterner();
+    private final Interner interner;
+
+    public static String intern(String sample) {
+        return INSTANCE.doIntern(sample);
+    }
+
+    private StringInterner() {
+        this.interner = Interners.newWeakInterner();
+    }
+
+    String doIntern(String sample) {
+        if (sample == null) {
+            return null;
+        }
+        return interner.intern(sample);
+    }
+}

From a26cf3eb78858ef5638650b6dbd7dab1f76a8c61 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 01:42:16 +0300
Subject: [PATCH 333/404] [fix][test] Fix flaky test
 NonPersistentTopicTest.testMsgDropStat, 3rd attempt (#20429)

---
 .../pulsar/client/api/NonPersistentTopicTest.java   | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index 63ce0f00dff15..4f64c4271fe89 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -40,6 +40,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
+import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.LoadManager;
@@ -821,7 +822,7 @@ public void testMsgDropStat() throws Exception {
 
         int defaultNonPersistentMessageRate = conf.getMaxConcurrentNonPersistentMessagePerConnection();
         try {
-            final String topicName = "non-persistent://my-property/my-ns/stats-topic";
+            final String topicName = BrokerTestUtil.newUniqueName("non-persistent://my-property/my-ns/stats-topic");
             // restart broker with lower publish rate limit
             conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
             stopBroker();
@@ -829,12 +830,15 @@ public void testMsgDropStat() throws Exception {
 
             pulsar.getBrokerService().updateRates();
 
+            @Cleanup
             Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
                     .receiverQueueSize(1).subscribe();
 
+            @Cleanup
             Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
                     .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
 
+            @Cleanup
             ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName)
                 .enableBatching(false)
                 .messageRoutingMode(MessageRoutingMode.SinglePartition)
@@ -861,12 +865,12 @@ public void testMsgDropStat() throws Exception {
                     });
                 });
             }
-            latch.await(5, TimeUnit.SECONDS);
+            assertTrue(latch.await(5, TimeUnit.SECONDS));
 
             NonPersistentTopic topic =
                     (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
 
-            Awaitility.await().untilAsserted(() -> {
+            Awaitility.await().ignoreExceptions().untilAsserted(() -> {
                 pulsar.getBrokerService().updateRates();
                 NonPersistentTopicStats stats = topic.getStats(false, false, false);
                 NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
@@ -877,9 +881,6 @@ public void testMsgDropStat() throws Exception {
                 assertTrue(sub2Stats.getMsgDropRate() > 0);
             });
 
-            producer.close();
-            consumer.close();
-            consumer2.close();
         } finally {
             conf.setMaxConcurrentNonPersistentMessagePerConnection(defaultNonPersistentMessageRate);
         }

From 7ea8741af265bd8554e7a52a29a4ed59cb749dea Mon Sep 17 00:00:00 2001
From: thetumbled <52550727+thetumbled@users.noreply.github.com>
Date: Tue, 30 May 2023 10:07:09 +0800
Subject: [PATCH 334/404] [Fix][Txn] Unwrap the completion exception. (#20396)

---
 .../pulsar/broker/TransactionMetadataStoreService.java       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
index 3e3b044ec51b8..35aa7cc2fdd26 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
@@ -169,7 +169,8 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc
                                     tcLoadSemaphore.release();
                                 })).exceptionally(e -> {
                             internalPinnedExecutor.execute(() -> {
-                                completableFuture.completeExceptionally(e.getCause());
+                                Throwable realCause = FutureUtil.unwrapCompletionException(e);
+                                completableFuture.completeExceptionally(realCause);
                                 // release before handle request queue,
                                 //in order to client reconnect infinite loop
                                 tcLoadSemaphore.release();
@@ -180,7 +181,7 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc
                                         CompletableFuture future = deque.poll();
                                         if (future != null) {
                                             // this means that this tc client connection connect fail
-                                            future.completeExceptionally(e);
+                                            future.completeExceptionally(realCause);
                                         } else {
                                             break;
                                         }

From e38091044c428af002b16110531497e2abc897d2 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 07:08:41 +0300
Subject: [PATCH 335/404] [improve][misc] Upgrade Netty to 4.1.93.Final
 (#20423)

---
 buildtools/pom.xml                            |  2 +-
 .../server/src/assemble/LICENSE.bin.txt       | 62 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        | 58 ++++++++---------
 pom.xml                                       |  4 +-
 pulsar-sql/presto-distribution/LICENSE        | 60 +++++++++---------
 5 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index 9e4ac022024e1..5a391777f2567 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -47,7 +47,7 @@
     4.1
     8.37
     3.1.2
-    4.1.89.Final
+    4.1.93.Final
     4.2.3
     31.0.1-jre
     1.10.12
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 220f0ac0758b8..487e4e96b6a66 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -289,37 +289,37 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.89.Final.jar
-    - io.netty-netty-codec-4.1.89.Final.jar
-    - io.netty-netty-codec-dns-4.1.89.Final.jar
-    - io.netty-netty-codec-http-4.1.89.Final.jar
-    - io.netty-netty-codec-http2-4.1.89.Final.jar
-    - io.netty-netty-codec-socks-4.1.89.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.89.Final.jar
-    - io.netty-netty-common-4.1.89.Final.jar
-    - io.netty-netty-handler-4.1.89.Final.jar
-    - io.netty-netty-handler-proxy-4.1.89.Final.jar
-    - io.netty-netty-resolver-4.1.89.Final.jar
-    - io.netty-netty-resolver-dns-4.1.89.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.89.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.89.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.89.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-epoll-4.1.89.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.89.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar
-    - io.netty-netty-tcnative-classes-2.0.56.Final.jar
-    - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.18.Final.jar
-    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar
-    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar
+    - io.netty-netty-buffer-4.1.93.Final.jar
+    - io.netty-netty-codec-4.1.93.Final.jar
+    - io.netty-netty-codec-dns-4.1.93.Final.jar
+    - io.netty-netty-codec-http-4.1.93.Final.jar
+    - io.netty-netty-codec-http2-4.1.93.Final.jar
+    - io.netty-netty-codec-socks-4.1.93.Final.jar
+    - io.netty-netty-codec-haproxy-4.1.93.Final.jar
+    - io.netty-netty-common-4.1.93.Final.jar
+    - io.netty-netty-handler-4.1.93.Final.jar
+    - io.netty-netty-handler-proxy-4.1.93.Final.jar
+    - io.netty-netty-resolver-4.1.93.Final.jar
+    - io.netty-netty-resolver-dns-4.1.93.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.1.93.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.1.93.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.1.93.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-epoll-4.1.93.Final.jar
+    - io.netty-netty-transport-native-unix-common-4.1.93.Final.jar
+    - io.netty-netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+    - io.netty-netty-tcnative-classes-2.0.61.Final.jar
+    - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.21.Final.jar
+    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar
+    - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar
  * Prometheus client
     - io.prometheus.jmx-collector-0.16.1.jar
     - io.prometheus-simpleclient-0.16.0.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 711890809f1bf..c04ac2b7d0363 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -348,35 +348,35 @@ The Apache Software License, Version 2.0
     - commons-text-1.10.0.jar
     - commons-compress-1.21.jar
  * Netty
-    - netty-buffer-4.1.89.Final.jar
-    - netty-codec-4.1.89.Final.jar
-    - netty-codec-dns-4.1.89.Final.jar
-    - netty-codec-http-4.1.89.Final.jar
-    - netty-codec-socks-4.1.89.Final.jar
-    - netty-codec-haproxy-4.1.89.Final.jar
-    - netty-common-4.1.89.Final.jar
-    - netty-handler-4.1.89.Final.jar
-    - netty-handler-proxy-4.1.89.Final.jar
-    - netty-resolver-4.1.89.Final.jar
-    - netty-resolver-dns-4.1.89.Final.jar
-    - netty-transport-4.1.89.Final.jar
-    - netty-transport-classes-epoll-4.1.89.Final.jar
-    - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.89.Final.jar
-    - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar
-    - netty-tcnative-classes-2.0.56.Final.jar
-    - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar
-    - netty-resolver-dns-classes-macos-4.1.89.Final.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar
+    - netty-buffer-4.1.93.Final.jar
+    - netty-codec-4.1.93.Final.jar
+    - netty-codec-dns-4.1.93.Final.jar
+    - netty-codec-http-4.1.93.Final.jar
+    - netty-codec-socks-4.1.93.Final.jar
+    - netty-codec-haproxy-4.1.93.Final.jar
+    - netty-common-4.1.93.Final.jar
+    - netty-handler-4.1.93.Final.jar
+    - netty-handler-proxy-4.1.93.Final.jar
+    - netty-resolver-4.1.93.Final.jar
+    - netty-resolver-dns-4.1.93.Final.jar
+    - netty-transport-4.1.93.Final.jar
+    - netty-transport-classes-epoll-4.1.93.Final.jar
+    - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.1.93.Final.jar
+    - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+    - netty-tcnative-classes-2.0.61.Final.jar
+    - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar
+    - netty-resolver-dns-classes-macos-4.1.93.Final.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
  * Prometheus client
     - simpleclient-0.16.0.jar
     - simpleclient_log4j2-0.16.0.jar
diff --git a/pom.xml b/pom.xml
index b45debcc9db25..c1a126b1161c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,8 +140,8 @@ flexible messaging model and an intuitive client API.
     1.1.8.4 
     4.1.12.1 
     5.1.0
-    4.1.89.Final
-    0.0.18.Final
+    4.1.93.Final
+    0.0.21.Final
     9.4.48.v20220622
     2.5.2
     2.34
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index a85d1bc363c1b..2a13985ac4ed5 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -231,37 +231,37 @@ The Apache Software License, Version 2.0
     - commons-compress-1.21.jar
     - commons-lang3-3.11.jar
  * Netty
-    - netty-buffer-4.1.89.Final.jar
-    - netty-codec-4.1.89.Final.jar
-    - netty-codec-dns-4.1.89.Final.jar
-    - netty-codec-http-4.1.89.Final.jar
-    - netty-codec-haproxy-4.1.89.Final.jar
-    - netty-codec-socks-4.1.89.Final.jar
-    - netty-handler-proxy-4.1.89.Final.jar
-    - netty-common-4.1.89.Final.jar
-    - netty-handler-4.1.89.Final.jar
+    - netty-buffer-4.1.93.Final.jar
+    - netty-codec-4.1.93.Final.jar
+    - netty-codec-dns-4.1.93.Final.jar
+    - netty-codec-http-4.1.93.Final.jar
+    - netty-codec-haproxy-4.1.93.Final.jar
+    - netty-codec-socks-4.1.93.Final.jar
+    - netty-handler-proxy-4.1.93.Final.jar
+    - netty-common-4.1.93.Final.jar
+    - netty-handler-4.1.93.Final.jar
     - netty-reactive-streams-2.0.6.jar
-    - netty-resolver-4.1.89.Final.jar
-    - netty-resolver-dns-4.1.89.Final.jar
-    - netty-resolver-dns-classes-macos-4.1.89.Final.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar
-    - netty-tcnative-classes-2.0.56.Final.jar
-    - netty-transport-4.1.89.Final.jar
-    - netty-transport-classes-epoll-4.1.89.Final.jar
-    - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.89.Final.jar
-    - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar
-    - netty-codec-http2-4.1.89.Final.jar
-    - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar
-    - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar
+    - netty-resolver-4.1.93.Final.jar
+    - netty-resolver-dns-4.1.93.Final.jar
+    - netty-resolver-dns-classes-macos-4.1.93.Final.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+    - netty-tcnative-classes-2.0.61.Final.jar
+    - netty-transport-4.1.93.Final.jar
+    - netty-transport-classes-epoll-4.1.93.Final.jar
+    - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.1.93.Final.jar
+    - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar
+    - netty-codec-http2-4.1.93.Final.jar
+    - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar
+    - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar
  * GRPC
     - grpc-api-1.45.1.jar
     - grpc-context-1.45.1.jar

From e0b953175aa39bbea2da967c753e696f45dd69e9 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 11:37:13 +0300
Subject: [PATCH 336/404] [fix][test] Improve deleteNamespaceWithRetry (#20431)

---
 .../broker/auth/MockedPulsarServiceBaseTest.java   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
index b688d5fbf24d8..c32d3fc3b0b27 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.pulsar.broker.auth;
 
-import static org.apache.pulsar.broker.BrokerTestUtil.*;
+import static org.apache.pulsar.broker.BrokerTestUtil.spyWithoutRecordingInvocations;
 import com.google.common.collect.Sets;
 import java.lang.reflect.Field;
 import java.net.InetSocketAddress;
@@ -55,11 +55,11 @@
 import org.apache.pulsar.tests.TestRetrySupport;
 import org.apache.pulsar.utils.ResourceUtils;
 import org.apache.zookeeper.MockZooKeeper;
+import org.awaitility.Awaitility;
 import org.mockito.Mockito;
 import org.mockito.internal.util.MockUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.DataProvider;
 
 /**
@@ -615,12 +615,18 @@ public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmi
      */
     public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin admin,
                                                 Collection pulsars) throws Exception {
-        Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> {
+        Awaitility.await()
+                .pollDelay(500, TimeUnit.MILLISECONDS)
+                .until(() -> {
             try {
                 // Maybe fail by race-condition with create topics, just retry.
                 admin.namespaces().deleteNamespace(ns, force);
                 return true;
-            } catch (Exception ex) {
+            } catch (PulsarAdminException.NotFoundException ex) {
+                // namespace was already deleted, ignore exception
+                return true;
+            } catch (Exception e) {
+                log.warn("Failed to delete namespace {} (force={})", ns, force, e);
                 return false;
             }
         });

From 639c460a585c2f6cbb6133849348c8b92d2f6fbd Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Tue, 30 May 2023 17:57:08 +0800
Subject: [PATCH 337/404] [fix][broker] Skip loading broker interceptor when
 disableBrokerInterceptors is true (#20422)

Signed-off-by: Zixuan Liu 
---
 .../pulsar/broker/ServiceConfiguration.java   |  2 +-
 .../broker/intercept/BrokerInterceptors.java  |  5 ++++
 .../intercept/BrokerInterceptorTest.java      | 25 +++++++++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 6fee0e7cd09d2..5dc15bd6a4dcf 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1347,7 +1347,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
 
     @FieldContext(
         category = CATEGORY_SERVER,
-        doc = "Enable or disable the broker interceptor, which is only used for testing for now"
+        doc = "Enable or disable the broker interceptor"
     )
     private boolean disableBrokerInterceptors = true;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java
index cef3f0eb609a1..4ffd8732db94e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java
@@ -59,6 +59,11 @@ public BrokerInterceptors(Map intercep
      * @return the collection of broker event interceptor
      */
     public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException {
+        if (conf.isDisableBrokerInterceptors()) {
+            log.info("Skip loading the broker interceptors when disableBrokerInterceptors is true");
+            return null;
+        }
+
         BrokerInterceptorDefinitions definitions =
                 BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(),
                         conf.getNarExtractionDirectory());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java
index d1cf91635f992..c612104f8bf1b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java
@@ -20,9 +20,12 @@
 
 import static org.mockito.ArgumentMatchers.same;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -36,6 +39,7 @@
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
+import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
@@ -307,4 +311,25 @@ public void requestInterceptorFailedTest() {
         }
     }
 
+    @Test
+    public void testLoadWhenDisableBrokerInterceptorsIsTrue() throws IOException {
+        ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class);
+        serviceConfiguration.setDisableBrokerInterceptors(true);
+        BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration);
+        assertNull(brokerInterceptor);
+
+        verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors();
+        verify(serviceConfiguration, times(0)).getBrokerInterceptorsDirectory();
+    }
+
+    @Test
+    public void testLoadWhenDisableBrokerInterceptorsIsFalse() throws IOException {
+        ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class);
+        serviceConfiguration.setDisableBrokerInterceptors(false);
+        BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration);
+        assertNull(brokerInterceptor);
+
+        verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors();
+        verify(serviceConfiguration, times(1)).getBrokerInterceptorsDirectory();
+    }
 }

From 5f2c29db4c78f062cedba4014eb459885ec3f1b7 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 30 May 2023 13:03:19 +0300
Subject: [PATCH 338/404] [fix][test] Replace usage of
 org.testcontainers.shaded.* classes in tests (#20437)

---
 .../src/main/resources/pulsar/suppressions.xml      |  2 +-
 .../broker/LedgerLostAndSkipNonRecoverableTest.java |  2 +-
 .../bucket/BucketDelayedDeliveryTrackerTest.java    |  6 +++---
 .../AntiAffinityNamespaceGroupExtensionTest.java    |  2 +-
 .../loadbalance/extensions/BrokerRegistryTest.java  |  3 +--
 .../extensions/ExtensibleLoadManagerImplTest.java   | 13 ++++++-------
 .../reporter/BrokerLoadDataReporterTest.java        |  2 +-
 .../reporter/TopBundleLoadDataReporterTest.java     |  2 +-
 .../transaction/SegmentAbortedTxnProcessorTest.java |  5 ++---
 .../apache/pulsar/client/impl/NegativeAcksTest.java |  3 +--
 10 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml
index 7c78988db3e90..57a01c60f6a27 100644
--- a/buildtools/src/main/resources/pulsar/suppressions.xml
+++ b/buildtools/src/main/resources/pulsar/suppressions.xml
@@ -38,7 +38,7 @@
     
 
     
-    
+    
 
     
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
index 389af8f2cd9f9..dbaaee8f48cd5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java
@@ -44,7 +44,7 @@
 import org.apache.pulsar.client.impl.BatchMessageIdImpl;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.util.FutureUtil;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 39b3992fbd195..1e3e72aa0ec44 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -46,14 +46,14 @@
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest;
 import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage;
 import org.apache.pulsar.broker.delayed.MockManagedCursor;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
+import org.awaitility.Awaitility;
 import org.roaringbitmap.RoaringBitmap;
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
-import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.DataProvider;
@@ -439,7 +439,7 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
 
         tracker.close();
     }
-    
+
     @Test(dataProvider = "delayedTracker")
     public void testClear(BucketDelayedDeliveryTracker tracker)
             throws ExecutionException, InterruptedException, TimeoutException {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
index 3469dbe5a5499..014e6fa3c7a04 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import lombok.Cleanup;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.loadbalance.AntiAffinityNamespaceGroupTest;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
@@ -42,7 +43,6 @@
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.common.naming.ServiceUnitId;
-import org.testcontainers.shaded.org.apache.commons.lang3.reflect.FieldUtils;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
index 26986a494f0be..fca41837b9df4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java
@@ -25,7 +25,6 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import java.time.Duration;
 import java.util.HashSet;
 import java.util.List;
@@ -54,7 +53,7 @@
 import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.awaitility.Awaitility;
-import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
+import org.awaitility.reflect.WhiteboxImpl;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 498f48d16d2cd..e8a4682e528d5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -48,10 +48,14 @@
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import com.google.common.collect.Sets;
+import java.net.URL;
 import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -59,11 +63,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
-import java.net.URL;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -101,7 +100,7 @@
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
index 93ab35981e1c4..9b0530349d036 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
@@ -45,9 +45,9 @@
 import org.apache.pulsar.client.util.ExecutorProvider;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import org.awaitility.Awaitility;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
index be8c6af2b0404..344387b293004 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
@@ -48,7 +48,7 @@
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.awaitility.Awaitility;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index 0600833b1adb6..32de5944daddf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -21,7 +21,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import java.lang.reflect.Field;
 import java.util.LinkedList;
 import java.util.NavigableMap;
@@ -58,8 +57,8 @@
 import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.TopicStats;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
-import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
+import org.awaitility.Awaitility;
+import org.awaitility.reflect.WhiteboxImpl;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java
index b4d01e263bc7a..0ae36b4ca9045 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java
@@ -21,7 +21,6 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
-
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
@@ -39,7 +38,7 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.TopicMessageId;
 import org.awaitility.Awaitility;
-import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl;
+import org.awaitility.reflect.WhiteboxImpl;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

From 35d961242089a0448cb425da637a2afbdaca0005 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Tue, 30 May 2023 18:07:31 +0800
Subject: [PATCH 339/404] [fix] [meta]Switch to the metadata store thread after
 zk operation (#20303)

---
 .../metadata/impl/AbstractMetadataStore.java  | 13 ++++
 .../pulsar/metadata/impl/ZKMetadataStore.java | 46 ++++++------
 .../pulsar/metadata/MetadataStoreTest.java    | 70 +++++++++++++++++++
 3 files changed, 106 insertions(+), 23 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
index 6fcf8eb6b49b2..4cadf2397a7fa 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
@@ -41,6 +41,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -534,6 +535,18 @@ public void execute(Runnable task, CompletableFuture future) {
         }
     }
 
+    /**
+     * Run the task in the executor thread and fail the future if the executor is shutting down.
+     */
+    @VisibleForTesting
+    public void execute(Runnable task, Supplier>> futures) {
+        try {
+            executor.execute(task);
+        } catch (final Throwable t) {
+            futures.get().forEach(f -> f.completeExceptionally(t));
+        }
+    }
+
     protected static String parent(String path) {
         int idx = path.lastIndexOf('/');
         if (idx <= 0) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
index a6d8eb8344c96..079ae3e2ae5c3 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
@@ -204,29 +204,29 @@ protected void batchOperation(List ops) {
                 }
 
                 // Trigger all the futures in the batch
-                for (int i = 0; i < ops.size(); i++) {
-                    OpResult opr = results.get(i);
-                    MetadataOp op = ops.get(i);
-
-                    switch (op.getType()) {
-                        case PUT:
-                            handlePutResult(op.asPut(), opr);
-                            break;
-                        case DELETE:
-                            handleDeleteResult(op.asDelete(), opr);
-                            break;
-                        case GET:
-                            handleGetResult(op.asGet(), opr);
-                            break;
-                        case GET_CHILDREN:
-                            handleGetChildrenResult(op.asGetChildren(), opr);
-                            break;
-
-                        default:
-                            op.getFuture().completeExceptionally(new MetadataStoreException(
-                                    "Operation type not supported in multi: " + op.getType()));
-                    }
-                }
+                execute(() -> {
+                    for (int i = 0; i < ops.size(); i++) {
+                        OpResult opr = results.get(i);
+                        MetadataOp op = ops.get(i);
+                            switch (op.getType()) {
+                                case PUT:
+                                    handlePutResult(op.asPut(), opr);
+                                    break;
+                                case DELETE:
+                                    handleDeleteResult(op.asDelete(), opr);
+                                    break;
+                                case GET:
+                                    handleGetResult(op.asGet(), opr);
+                                    break;
+                                case GET_CHILDREN:
+                                    handleGetChildrenResult(op.asGetChildren(), opr);
+                                    break;
+                                default:
+                                    op.getFuture().completeExceptionally(new MetadataStoreException(
+                                            "Operation type not supported in multi: " + op.getType()));
+                            }
+                        }
+                }, () -> ops.stream().map(MetadataOp::getFuture).collect(Collectors.toList()));
             }, null);
         } catch (Throwable t) {
             ops.forEach(o -> o.getFuture().completeExceptionally(new MetadataStoreException(t)));
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
index 949b4a9b2bacb..c87e9bda436bf 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -50,8 +51,10 @@
 import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.NotificationType;
 import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
 import org.assertj.core.util.Lists;
 import org.awaitility.Awaitility;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Slf4j
@@ -425,6 +428,73 @@ public void testDeleteUnusedDirectories(String provider, Supplier urlSup
         assertFalse(store.exists(prefix).join());
     }
 
+    @DataProvider(name = "conditionOfSwitchThread")
+    public Object[][] conditionOfSwitchThread(){
+        return new Object[][]{
+            {false, false},
+            {false, true},
+            {true, false},
+            {true, true}
+        };
+    }
+
+    @Test(dataProvider = "conditionOfSwitchThread")
+    public void testThreadSwitchOfZkMetadataStore(boolean hasSynchronizer, boolean enabledBatch) throws Exception {
+        final String prefix = newKey();
+        final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", "");
+        MetadataStoreConfig.MetadataStoreConfigBuilder builder =
+                MetadataStoreConfig.builder().metadataStoreName(metadataStoreName);
+        builder.fsyncEnable(false);
+        builder.batchingEnabled(enabledBatch);
+        if (!hasSynchronizer) {
+            builder.synchronizer(null);
+        }
+        MetadataStoreConfig config = builder.build();
+        @Cleanup
+        ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config);
+
+        final Runnable verify = () -> {
+            String currentThreadName = Thread.currentThread().getName();
+            String errorMessage = String.format("Expect to switch to thread %s, but currently it is thread %s",
+                    metadataStoreName, currentThreadName);
+            assertTrue(Thread.currentThread().getName().startsWith(metadataStoreName), errorMessage);
+        };
+
+        // put with node which has parent(but the parent node is not exists).
+        store.put(prefix + "/a1/b1/c1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // put.
+        store.put(prefix + "/b1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // get.
+        store.get(prefix + "/b1").thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // get the node which is not exists.
+        store.get(prefix + "/non").thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // delete.
+        store.delete(prefix + "/b1", Optional.empty()).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).join();
+        // delete the node which is not exists.
+        store.delete(prefix + "/non", Optional.empty()).thenApply((ignore) -> {
+            verify.run();
+            return null;
+        }).exceptionally(ex -> {
+            verify.run();
+            return null;
+        }).join();
+    }
+
     @Test(dataProvider = "impl")
     public void testPersistent(String provider, Supplier urlSupplier) throws Exception {
         String metadataUrl = urlSupplier.get();

From 113d70d9299d5bc893cee9b524c7b1623dee5892 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= 
Date: Tue, 30 May 2023 18:46:56 +0800
Subject: [PATCH 340/404] [improve][test] Add unit test for metadata cache
 #20363 (#20436)

- Add `MetadataStoreTest#testExistsDistributed` for distributed metaStore implementations only
- Add `MetadataStoreTest#testGetChildrenDistributed` for distributed metaStore implementations only
---
 .../metadata/BaseMetadataStoreTest.java       |  8 +++
 .../pulsar/metadata/MetadataStoreTest.java    | 52 +++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
index ec6e6e03eae71..411ee038c48b0 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
@@ -82,6 +82,14 @@ public Object[][] implementations() {
         };
     }
 
+    @DataProvider(name = "distributedImpl")
+    public Object[][] distributedImplementations() {
+        return new Object[][]{
+                {"ZooKeeper", stringSupplier(() -> zks.getConnectionString())},
+                {"Etcd", stringSupplier(() -> "etcd:" + getEtcdClusterConnectString())},
+        };
+    }
+
     private synchronized String getEtcdClusterConnectString() {
         if (etcdCluster == null) {
             etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build()
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
index c87e9bda436bf..246661edc43ee 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
@@ -663,4 +663,56 @@ public void testClosedMetadataStore(String provider, Supplier urlSupplie
             assertTrue(e.getCause() instanceof MetadataStoreException.AlreadyClosedException);
         }
     }
+
+    @Test(dataProvider = "distributedImpl")
+    public void testGetChildrenDistributed(String provider, Supplier urlSupplier) throws Exception {
+        @Cleanup
+        MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+        @Cleanup
+        MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+        String parent = newKey();
+        byte[] value = "value1".getBytes(StandardCharsets.UTF_8);
+        store1.put(parent, value, Optional.empty()).get();
+        store1.put(parent + "/a", value, Optional.empty()).get();
+        assertEquals(store1.getChildren(parent).get(), List.of("a"));
+        store1.delete(parent + "/a", Optional.empty()).get();
+        assertEquals(store1.getChildren(parent).get(), Collections.emptyList());
+        store1.delete(parent, Optional.empty()).get();
+        assertEquals(store1.getChildren(parent).get(), Collections.emptyList());
+        store2.put(parent + "/b", value, Optional.empty()).get();
+        // There is a chance watcher event is not triggered before the store1.getChildren() call.
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> assertEquals(store1.getChildren(parent).get(), List.of("b")));
+        store2.put(parent + "/c", value, Optional.empty()).get();
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> assertEquals(store1.getChildren(parent).get(), List.of("b", "c")));
+    }
+
+    @Test(dataProvider = "distributedImpl")
+    public void testExistsDistributed(String provider, Supplier urlSupplier) throws Exception {
+        @Cleanup
+        MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+        @Cleanup
+        MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(),
+                MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+        String parent = newKey();
+        byte[] value = "value1".getBytes(StandardCharsets.UTF_8);
+        assertFalse(store1.exists(parent).get());
+        store1.put(parent, value, Optional.empty()).get();
+        assertTrue(store1.exists(parent).get());
+        assertFalse(store1.exists(parent + "/a").get());
+        store2.put(parent + "/a", value, Optional.empty()).get();
+        assertTrue(store1.exists(parent + "/a").get());
+        // There is a chance watcher event is not triggered before the store1.exists() call.
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> assertFalse(store1.exists(parent + "/b").get()));
+    }
 }

From fb05c4559c83023a00ceb097c994a088a050c03d Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Tue, 30 May 2023 15:00:41 +0300
Subject: [PATCH 341/404] [improve][misc] PIP-265: Delete wiki/proposals and
 define new process in new folder (#20418)

---
 pip/README.md                  |  97 ++++++++++++++++++++
 pip/TEMPLATE.md                | 163 +++++++++++++++++++++++++++++++++
 wiki/proposals/PIP-template.md |  91 ------------------
 wiki/proposals/PIP.md          | 120 ------------------------
 4 files changed, 260 insertions(+), 211 deletions(-)
 create mode 100644 pip/README.md
 create mode 100644 pip/TEMPLATE.md
 delete mode 100644 wiki/proposals/PIP-template.md
 delete mode 100644 wiki/proposals/PIP.md

diff --git a/pip/README.md b/pip/README.md
new file mode 100644
index 0000000000000..3ed9a1d34cd1d
--- /dev/null
+++ b/pip/README.md
@@ -0,0 +1,97 @@
+# Pulsar Improvement Proposal (PIP)
+
+## What is a PIP?
+
+The PIP is a "Pulsar Improvement Proposal" and it's the mechanism used to propose changes to the Apache Pulsar codebases.
+
+The changes might be in terms of new features, large code refactoring, changes to APIs.
+
+In practical terms, the PIP defines a process in which developers can submit a design doc, receive feedback and get the "go ahead" to execute.
+
+### What is the goal of a PIP?
+
+There are several goals for the PIP process:
+
+1. Ensure community technical discussion of major changes to the Apache Pulsar codebase.
+
+2. Provide clear and thorough design documentation of the proposed changes. Make sure every Pulsar developer will have enough context to effectively perform a code review of the Pull Requests.
+
+3. Use the PIP document to serve as the baseline on which to create the documentation for the new feature.
+
+4. Have greater scrutiny to changes are affecting the public APIs (as defined below) to reduce chances of introducing breaking changes or APIs that are not expressing an ideal semantic.
+
+It is not a goal for PIP to add undue process or slow-down the development.
+
+### When is a PIP required?
+
+* Any new feature for Pulsar brokers or client
+* Any change to the public APIs (Client APIs, REST APIs, Plugin APIs)
+* Any change to the wire protocol APIs
+* Any change to the API of Pulsar CLI tools (eg: new options)
+* Any change to the semantic of existing functionality, even when current behavior is incorrect.
+* Any large code change that will touch multiple components
+* Any changes to the metrics (metrics endpoint, topic stats, topics internal stats, broker stats, etc.)
+* Any change to the configuration
+
+### When is a PIP *not* required?
+
+* Bug-fixes
+* Simple enhancements that won't affect the APIs or the semantic
+* Small documentation changes
+* Small website changes
+* Build scripts changes (except: a complete rewrite)
+
+### Who can create a PIP?
+
+Any person willing to contribute to the Apache Pulsar project is welcome to create a PIP.
+
+## How does the PIP process work?
+
+A PIP proposal can be in these states:
+1. **DRAFT**: (Optional) This might be used for contributors to collaborate and to seek feedback on an incomplete version of the proposal.
+
+2. **DISCUSSION**: The proposal has been submitted to the community for feedback and approval.
+
+3. **ACCEPTED**: The proposal has been accepted by the Pulsar project.
+
+4. **REJECTED**: The proposal has not been accepted by the Pulsar project.
+
+5. **IMPLEMENTED**: The implementation of the proposed changes have been completed and everything has been merged.
+
+6. **RELEASED**: The proposed changes have been included in an official
+   Apache Pulsar release.
+
+
+The process works in the following way:
+
+1. Fork https://github.com/apache/pulsar repository (Using the fork button on GitHub).
+2. Clone the repository, and on it, copy the file `pip/TEMPLATE.md` and name it `pip-xxx.md`. The number `xxx` should be the next sequential number after the last contributed PIP. You view the list of contributed PIPs (at any status) as a list of Pull Requests having a title starting with `[pip][design] PIP-`. Use the link [here](https://github.com/apache/pulsar/pulls?q=is%3Apr+title%3A%22%5Bpip%5D%5Bdesign%5D+PIP-%22) as shortcut.
+3. Write the proposal following the section outlined by the template and the explanation for each section in the comment it contains (you can delete the comment once done).
+   * If you need diagrams, avoid attaching large files. You can use [MermaidJS](https://mermaid.js.org/) as simple language to describe many types of diagrams. 
+4. Create GitHub Pull request (PR). The PR title should be `[pip][design] PIP-xxx: {title}`, where the `xxx` match the number given in previous step (file-name). Replace `{title}` with a short title to your proposal.
+5. The author(s) will email the dev@pulsar.apache.org mailing list to kick off a discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion will happen in broader context either on the mailing list or as general comments on the PR. Many of the discussion items will be on particular aspect of the proposal, hence they should be as comments in the PR to specific lines in the proposal file.
+6. Update file with a link to the discussion on the mailing. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org).
+7. Based on the discussion and feedback, some changes might be applied by authors to the text of the proposal. They will be applied as extra commits, making it easier to track the changes.
+8. Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list, by
+   sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}`.
+   Make sure to update the PIP with a link to the vote. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org). 
+   Everyone is welcome to vote on the proposal, though only the vote of the PMC members will be considered binding.
+   It is required to have a lazy majority of at least 3 binding +1s votes.
+   The vote should stay open for at least 48 hours.
+9. When the vote is closed, if the outcome is positive, ask a PMC member (using voting thread on mailing list) to merge the PR.
+10. If the outcome is negative, please close the PR (with a small comment that the close is a result of a vote).
+
+All the future implementation Pull Requests that will be created, should always reference the PIP-XXX in the commit log message and the PR title.
+It is advised to create a master GitHub issue to formulate the execution plan and track its progress.
+
+## List of PIPs
+
+### Historical PIPs
+You can the view list of PIPs previously managed by GitHub wiki or GitHub issues [here](https://github.com/apache/pulsar/wiki#pulsar-improvement-proposals)
+
+### List of PIPs
+1. You can view all PIPs (besides the historical ones) as the list of Pull Requests having title starting with `[pip][design] PIP-`. Here is the [link](https://github.com/apache/pulsar/pulls?q=is%3Apr+title%3A%22%5Bpip%5D%5Bdesign%5D+PIP-%22) for it. 
+   - Merged PR means the PIP was accepted.
+   - Closed PR means the PIP was rejected.
+   - Open PR means the PIP was submitted and is in the process of discussion.
+2. You can also take a look at the file in the `pip` folder. Each one is an approved PIP.
\ No newline at end of file
diff --git a/pip/TEMPLATE.md b/pip/TEMPLATE.md
new file mode 100644
index 0000000000000..6f907eef7e8e9
--- /dev/null
+++ b/pip/TEMPLATE.md
@@ -0,0 +1,163 @@
+
+
+# Background knowledge
+
+
+
+# Motivation
+
+
+
+# Goals
+
+## In Scope
+
+
+
+## Out of Scope
+
+
+
+
+# High Level Design
+
+
+
+# Detailed Design
+
+## Design & Implementation Details
+
+
+
+## Public-facing Changes
+
+
+
+### Public API
+
+
+### Binary protocol
+
+### Configuration
+
+### CLI
+
+### Metrics
+
+
+
+
+# Monitoring
+
+
+
+# Security Considerations
+
+
+# Backward & Forward Compatability
+
+## Revert
+
+
+
+## Upgrade
+
+
+
+# Alternatives
+
+
+
+# General Notes
+
+# Links
+
+
+* Mailing List discussion thread:
+* Mailing List voting thread:
diff --git a/wiki/proposals/PIP-template.md b/wiki/proposals/PIP-template.md
deleted file mode 100644
index 752878a56b519..0000000000000
--- a/wiki/proposals/PIP-template.md
+++ /dev/null
@@ -1,91 +0,0 @@
-# Pulsar Improvement Proposal Template
-
-* **Status**: "one of ['Under Discussion', 'Accepted', 'Adopted', 'Rejected']"
-* **Author**: (Names)
-* **Pull Request**: (Link to the main pull request to resolve this PIP)
-* **Mailing List discussion**: (Link to the mailing list discussion)
-* **Release**: (Which release include this PIP)
-
-### Motivation
-
-_Describe the problems you are trying to solve_
-
-### Public Interfaces
-
-_Briefly list any new interfaces that will be introduced as part of this proposal or any existing interfaces that will be removed or changed. The purpose of this section is to concisely call out the public contract that will come along with this feature._
-
-A public interface is any change to the following:
-
-- Data format, Metadata format
-- The wire protocol and API behavior
-- Any class in the public packages
-- Monitoring
-- Command-line tools and arguments
-- Configuration settings
-- Anything else that will likely break existing users in some way when they upgrade
-
-### Proposed Changes
-
-_Describe the new thing you want to do in appropriate detail. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences. Use judgment based on the scope of the change._
-
-### Compatibility, Deprecation, and Migration Plan
-
-- What impact (if any) will there be on existing users?
-- If we are changing behavior how will we phase out the older behavior?
-- If we need special migration tools, describe them here.
-- When will we remove the existing behavior?
-
-### Test Plan
-
-_Describe in few sentences how the BP will be tested. We are mostly interested in system tests (since unit-tests are specific to implementation details). How will we know that the implementation works as expected? How will we know nothing broke?_
-
-### Rejected Alternatives
-
-_If there are alternative ways of accomplishing the same thing, what were they? The purpose of this section is to motivate why the design is the way it is and not some other way._
-
----
-
-```
-* **Status**: "one of ['Under Discussion', 'Accepted', 'Adopted', 'Rejected']"
-* **Author**: (Names)
-* **Pull Request**: (Link to the main pull request to resolve this PIP)
-* **Mailing List discussion**: (Link to the mailing list discussion)
-* **Release**: (Which release include this PIP)
-
-### Motivation
-
-_Describe the problems you are trying to solve_
-
-### Public Interfaces
-
-_Briefly list any new interfaces that will be introduced as part of this proposal or any existing interfaces that will be removed or changed. The purpose of this section is to concisely call out the public contract that will come along with this feature._
-
-A public interface is any change to the following:
-
-- Data format, Metadata format
-- The wire protocol and API behavior
-- Any class in the public packages
-- Monitoring
-- Command-line tools and arguments
-- Configuration settings
-- Anything else that will likely break existing users in some way when they upgrade
-
-### Proposed Changes
-
-_Describe the new thing you want to do in appropriate detail. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences. Use judgment based on the scope of the change._
-
-### Compatibility, Deprecation, and Migration Plan
-
-- What impact (if any) will there be on existing users? 
-- If we are changing behavior how will we phase out the older behavior? 
-- If we need special migration tools, describe them here.
-- When will we remove the existing behavior?
-
-### Test Plan
-
-_Describe in few sentences how the BP will be tested. We are mostly interested in system tests (since unit-tests are specific to implementation details). How will we know that the implementation works as expected? How will we know nothing broke?_
-
-### Rejected Alternatives
-
-_If there are alternative ways of accomplishing the same thing, what were they? The purpose of this section is to motivate why the design is the way it is and not some other way._
-```
\ No newline at end of file
diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md
deleted file mode 100644
index fc68905726eb8..0000000000000
--- a/wiki/proposals/PIP.md
+++ /dev/null
@@ -1,120 +0,0 @@
-# Pulsar Improvement Proposal (PIP)
-
-## What is a PIP?
-
-The PIP is a "Pulsar Improvement Proposal" and it's the mechanism used to
-propose changes to the Apache Pulsar codebases.
-
-The changes might be in terms of new features, large code refactoring, changes
-to APIs.
-
-In practical terms, the PIP defines a process in which developers can submit
-a design doc, receive feedback and get the "go ahead" to execute.
-
-## What is the goal of a PIP?
-
-There are several goals for the PIP process:
-
-1. Ensure community technical discussion of major changes to the Apache Pulsar
-   codebase
-
-2. Provide clear and thorough design documentation of the proposed changes.
-   Make sure every Pulsar developer will have enough context to effectively
-   perform a code review of the Pull Requests.
-
-3. Use the PIP document to serve as the starting base on which to create the
-   documentation for the new feature.
-
-4. Have greater scrutiny to changes are affecting the public APIs to reduce
-   chances of introducing breaking changes or APIs that are not expressing
-   an ideal semantic.
-
-
-It is not a goal for PIP to add undue process or slow-down the development.
-
-## When is a PIP required?
-
-* Any new feature for Pulsar brokers or client
-* Any change to the public APIs (Client APIs, REST APIs, Plugin APIs)
-* Any change to the wire protocol APIs
-* Any change to the API of Pulsar CLI tools (eg: new options)
-* Any change to the semantic of existing functionality, even when current
-  behavior is incorrect.
-* Any large code change that will touch multiple components
-* Any changes to the metrics (metrics endpoint, topic stats, topics internal stats, broker stats, etc.)
-
-## When is a PIP *not* required?
-
-* Bug-fixes
-* Simple enhancements that won't affect the APIs or the semantic
-* Documentation changes
-* Website changes
-* Build scripts changes (except: a complete rewrite)
-
-## Who can create a PIP?
-
-Any person willing to contribute to the Apache Pulsar project is welcome to
-create a PIP.
-
-## How does the PIP process work?
-
-A PIP proposal can be in these states:
-1. **DRAFT**: (Optional) This might be used for contributors to collaborate and
-   to seek feedback on an incomplete version of the proposal.
-
-2. **DISCUSSION**: The proposal has been submitted to the community for
-   feedback and approval.
-
-3. **ACCEPTED**: The proposal has been accepted by the Pulsar project.
-
-4. **REJECTED**: The proposal has not been accepted by the Pulsar project.
-
-5. **IMPLEMENTED**: The implementation of the proposed changes have been
-   completed and everything has been merged.
-
-5. **RELEASED**: The proposed changes have been included in an official
-   Apache Pulsar release.
-
-The process works in the following way:
-
-1. The author(s) of the proposal will create a GitHub issue ticket choosing the
-   template for PIP proposals. The issue title should be "PIP-xxx: title", where
-   the "xxx" number should be chosen to be the next number from the existing PIP 
-   issues, listed [here](https://github.com/apache/pulsar/issues?q=is%3Aissue+label%3APIP+)
-2. The author(s) will send a note to the dev@pulsar.apache.org mailing list
-   to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion
-   need to happen in the mailing list. Please avoid discussing it using
-   GitHub comments in the PIP GitHub issue, as it creates two tracks 
-   of feedback.
-3. Based on the discussion and feedback, some changes might be applied by
-   authors to the text of the proposal.
-4. Once some consensus is reached, there will be a vote to formally approve
-   the proposal.
-   The vote will be held on the dev@pulsar.apache.org mailing list, by
-   sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}".
-   Everyone is welcome to vote on the proposal, though only the the vote of the PMC 
-   members will be considered binding.
-   It is required to have a lazy majority of at least 3 binding +1s votes.
-   The vote should stay open for at least 48 hours.
-5. When the vote is closed, if the outcome is positive, the state of the
-   proposal is updated, and the Pull Requests associated with this proposal can
-   start to get merged into the master branch.
-
-All the Pull Requests that are created, should always reference the
-PIP-XXX in the
-commit log message and the PR title.
-
-## Labels of a PIP
-
-In addition to its current state, the GitHub issue for the PIP will also be
-tagged with other labels.
-
-Some examples:
-* Execution status: In progress, Completed, Need Help, ...
-* Targeted Pulsar release: 2.9, 2.10, ...
-
-
-## Template for a PIP design doc
-
-Read [the template file](/.github/ISSUE_TEMPLATE/pip.md).
-

From 70ea994316e5111bfdf995e8164f7f3a7ba0bd2b Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Tue, 30 May 2023 16:48:36 +0300
Subject: [PATCH 342/404] [improve][misc]  Deprecate PIP issue type (#20439)

---
 .github/ISSUE_TEMPLATE/pip.md    | 165 +------------------------------
 .github/PULL_REQUEST_TEMPLATE.md |   2 +-
 2 files changed, 5 insertions(+), 162 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/pip.md b/.github/ISSUE_TEMPLATE/pip.md
index 520826f8b1ceb..4274f2f637936 100644
--- a/.github/ISSUE_TEMPLATE/pip.md
+++ b/.github/ISSUE_TEMPLATE/pip.md
@@ -1,167 +1,10 @@
 ---
 name: PIP
-about: Submit a Pulsar Improvement Proposal (PIP)
+about: [DEPRECATED. see 'pip' folder] Submit a Pulsar Improvement Proposal (PIP)
+description: DEPRECATED. Please read `pip/README.md`
 title: 'PIP-XYZ: '
 labels: PIP
 ---
 
-
-
-# Background knowledge
-
-
-
-# Motivation
-
-
-
-# Goals
-
-## In Scope
-
-
-
-## Out of Scope
-
-
-
-
-# High Level Design
-
-
-
-# Detailed Design
-
-## Design & Implementation Details
-
-
-
-## Public-facing Changes
-
-
-
-### Public API 
-
-
-### Binary protocol
-
-### Configuration
-
-### CLI
-
-### Metrics
-
-
-
-
-# Monitoring
-
-
-
-# Security Considerations
-
-
-# Backward & Forward Compatability
-
-## Revert
-
-
-
-## Upgrade
-
-
-
-# Alternatives
-
-
-
-# General Notes
-
-# Links
-
-
-* Mailing List discussion thread:
-* Mailing List voting thread:
+We have stopped using GitHub issues to hold the PIP content.
+Please read [here](https://github.com/apache/pulsar/blob/master/pip/README.md) how to submit a PIP
\ No newline at end of file
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 01ac26570b2d1..fdb8459024b1f 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -24,7 +24,7 @@ Master Issue: #xyz
 
 PIP: #xyz 
 
-
+
 
 ### Motivation
 

From 5796842dd2a1dbec11101d14a742c729ef22ed5a Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Tue, 30 May 2023 17:05:16 +0300
Subject: [PATCH 343/404] [fix][misc] Fix invalid PIP GitHub issue template
 (#20440)

---
 .github/ISSUE_TEMPLATE/pip.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/pip.md b/.github/ISSUE_TEMPLATE/pip.md
index 4274f2f637936..e0bc586669493 100644
--- a/.github/ISSUE_TEMPLATE/pip.md
+++ b/.github/ISSUE_TEMPLATE/pip.md
@@ -1,10 +1,9 @@
 ---
 name: PIP
-about: [DEPRECATED. see 'pip' folder] Submit a Pulsar Improvement Proposal (PIP)
-description: DEPRECATED. Please read `pip/README.md`
-title: 'PIP-XYZ: '
+about: '[DEPRECATED. see pip folder] Submit a Pulsar Improvement Proposal (PIP)'
+title: 'DEPRECATED - Read https://github.com/apache/pulsar/blob/master/pip/README.md'
 labels: PIP
 ---
 
 We have stopped using GitHub issues to hold the PIP content.
-Please read [here](https://github.com/apache/pulsar/blob/master/pip/README.md) how to submit a PIP
\ No newline at end of file
+Please read [here](https://github.com/apache/pulsar/blob/master/pip/README.md) how to submit a PIP

From ad75cc87e045ffca56a44f69e189f2e080373cdb Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Wed, 31 May 2023 09:45:53 +0800
Subject: [PATCH 344/404] [improve][meta] Support set metadata size threshold
 for compression. (#19561)

---
 .../mledger/ManagedLedgerFactoryConfig.java   | 22 +++++
 .../mledger/MetadataCompressionConfig.java    | 56 +++++++++++++
 .../impl/ManagedLedgerFactoryImpl.java        |  5 +-
 .../mledger/impl/MetaStoreImpl.java           | 81 +++++++++----------
 .../impl/ManagedCursorInfoMetadataTest.java   | 57 +++++++++++--
 .../impl/ManagedLedgerInfoMetadataTest.java   | 63 +++++++++++++--
 .../pulsar/broker/ServiceConfiguration.java   | 13 +++
 .../broker/ManagedLedgerClientFactory.java    |  4 +
 8 files changed, 244 insertions(+), 57 deletions(-)
 create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
index 5aa4e8374d73a..8a4b4d4013f8f 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
@@ -92,8 +92,30 @@ public class ManagedLedgerFactoryConfig {
      */
     private String managedLedgerInfoCompressionType = MLDataFormats.CompressionType.NONE.name();
 
+    /**
+     * ManagedLedgerInfo compression threshold. If the origin metadata size below configuration.
+     * compression will not apply.
+     */
+    private long managedLedgerInfoCompressionThresholdInBytes = 0;
+
     /**
      * ManagedCursorInfo compression type. If the compression type is null or invalid, don't compress data.
      */
     private String managedCursorInfoCompressionType = MLDataFormats.CompressionType.NONE.name();
+
+    /**
+     * ManagedCursorInfo compression threshold. If the origin metadata size below configuration.
+     * compression will not apply.
+     */
+    private long managedCursorInfoCompressionThresholdInBytes = 0;
+
+    public MetadataCompressionConfig getCompressionConfigForManagedLedgerInfo() {
+        return new MetadataCompressionConfig(managedLedgerInfoCompressionType,
+                managedLedgerInfoCompressionThresholdInBytes);
+    }
+
+    public MetadataCompressionConfig getCompressionConfigForManagedCursorInfo() {
+        return new MetadataCompressionConfig(managedCursorInfoCompressionType,
+                managedCursorInfoCompressionThresholdInBytes);
+    }
 }
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java
new file mode 100644
index 0000000000000..601c270ab7680
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.commons.lang.StringUtils;
+
+@Data
+@AllArgsConstructor
+@ToString
+public class MetadataCompressionConfig {
+    MLDataFormats.CompressionType compressionType;
+    long compressSizeThresholdInBytes;
+
+    public MetadataCompressionConfig(String compressionType) throws IllegalArgumentException {
+        this(compressionType, 0);
+    }
+
+    public MetadataCompressionConfig(String compressionType, long compressThreshold) throws IllegalArgumentException {
+        this.compressionType = parseCompressionType(compressionType);
+        this.compressSizeThresholdInBytes = compressThreshold;
+    }
+
+    public static MetadataCompressionConfig noCompression =
+            new MetadataCompressionConfig(MLDataFormats.CompressionType.NONE, 0);
+
+    private MLDataFormats.CompressionType parseCompressionType(String value) throws IllegalArgumentException {
+        if (StringUtils.isEmpty(value)) {
+            return MLDataFormats.CompressionType.NONE;
+        }
+
+        MLDataFormats.CompressionType compressionType;
+        compressionType = MLDataFormats.CompressionType.valueOf(value);
+
+        return compressionType;
+    }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index 9f3fe9bb0c4a7..f076f68299dd0 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -193,8 +193,9 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
         this.bookkeeperFactory = bookKeeperGroupFactory;
         this.isBookkeeperManaged = isBookkeeperManaged;
         this.metadataStore = metadataStore;
-        this.store = new MetaStoreImpl(metadataStore, scheduledExecutor, config.getManagedLedgerInfoCompressionType(),
-                config.getManagedCursorInfoCompressionType());
+        this.store = new MetaStoreImpl(metadataStore, scheduledExecutor,
+                config.getCompressionConfigForManagedLedgerInfo(),
+                config.getCompressionConfigForManagedCursorInfo());
         this.config = config;
         this.mbean = new ManagedLedgerFactoryMBeanImpl(this);
         this.entryCacheManager = new RangeEntryCacheManagerImpl(this);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index 1bc2d2b04bed0..d9269ec83b179 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -37,11 +37,11 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
 import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException;
+import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo;
-import org.apache.commons.lang.StringUtils;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.compression.CompressionCodec;
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
@@ -62,50 +62,35 @@ public class MetaStoreImpl implements MetaStore, Consumer {
     private final OrderedExecutor executor;
 
     private static final int MAGIC_MANAGED_INFO_METADATA = 0x4778; // 0100 0111 0111 1000
-    private final CompressionType ledgerInfoCompressionType;
-    private final CompressionType cursorInfoCompressionType;
+    private final MetadataCompressionConfig ledgerInfoCompressionConfig;
+    private final MetadataCompressionConfig cursorInfoCompressionConfig;
 
     private final Map> managedLedgerInfoUpdateCallbackMap;
 
     public MetaStoreImpl(MetadataStore store, OrderedExecutor executor) {
         this.store = store;
         this.executor = executor;
-        this.ledgerInfoCompressionType = CompressionType.NONE;
-        this.cursorInfoCompressionType = CompressionType.NONE;
+        this.ledgerInfoCompressionConfig = MetadataCompressionConfig.noCompression;
+        this.cursorInfoCompressionConfig = MetadataCompressionConfig.noCompression;
         managedLedgerInfoUpdateCallbackMap = new ConcurrentHashMap<>();
         if (store != null) {
             store.registerListener(this);
         }
     }
 
-    public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String ledgerInfoCompressionType,
-                         String cursorInfoCompressionType) {
+    public MetaStoreImpl(MetadataStore store, OrderedExecutor executor,
+                         MetadataCompressionConfig ledgerInfoCompressionConfig,
+                         MetadataCompressionConfig cursorInfoCompressionConfig) {
         this.store = store;
         this.executor = executor;
-        this.ledgerInfoCompressionType = parseCompressionType(ledgerInfoCompressionType);
-        this.cursorInfoCompressionType = parseCompressionType(cursorInfoCompressionType);
+        this.ledgerInfoCompressionConfig = ledgerInfoCompressionConfig;
+        this.cursorInfoCompressionConfig = cursorInfoCompressionConfig;
         managedLedgerInfoUpdateCallbackMap = new ConcurrentHashMap<>();
         if (store != null) {
             store.registerListener(this);
         }
     }
 
-    private CompressionType parseCompressionType(String value) {
-        if (StringUtils.isEmpty(value)) {
-            return CompressionType.NONE;
-        }
-
-        CompressionType compressionType;
-        try {
-            compressionType = CompressionType.valueOf(value);
-        } catch (Exception e) {
-            log.error("Failed to get compression type {} error msg: {}.", value, e.getMessage());
-            throw e;
-        }
-
-        return compressionType;
-    }
-
     @Override
     public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map properties,
             MetaStoreCallback callback) {
@@ -421,29 +406,43 @@ private static MetaStoreException getException(Throwable t) {
     }
 
     public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) {
-        if (ledgerInfoCompressionType.equals(CompressionType.NONE)) {
+        CompressionType compressionType = ledgerInfoCompressionConfig.getCompressionType();
+        if (compressionType.equals(CompressionType.NONE)) {
             return managedLedgerInfo.toByteArray();
         }
-        MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata
-                .newBuilder()
-                .setCompressionType(ledgerInfoCompressionType)
-                .setUncompressedSize(managedLedgerInfo.getSerializedSize())
-                .build();
-        return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(),
-                mlInfoMetadata.getSerializedSize(), ledgerInfoCompressionType);
+
+        int uncompressedSize = managedLedgerInfo.getSerializedSize();
+        if (uncompressedSize > ledgerInfoCompressionConfig.getCompressSizeThresholdInBytes()) {
+            MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata
+                    .newBuilder()
+                    .setCompressionType(compressionType)
+                    .setUncompressedSize(uncompressedSize)
+                    .build();
+            return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(),
+                    mlInfoMetadata.getSerializedSize(), compressionType);
+        }
+
+        return managedLedgerInfo.toByteArray();
     }
 
     public byte[] compressCursorInfo(ManagedCursorInfo managedCursorInfo) {
-        if (cursorInfoCompressionType.equals(CompressionType.NONE)) {
+        CompressionType compressionType = cursorInfoCompressionConfig.getCompressionType();
+        if (compressionType.equals(CompressionType.NONE)) {
             return managedCursorInfo.toByteArray();
         }
-        MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata
-                .newBuilder()
-                .setCompressionType(cursorInfoCompressionType)
-                .setUncompressedSize(managedCursorInfo.getSerializedSize())
-                .build();
-        return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(),
-                metadata.getSerializedSize(), cursorInfoCompressionType);
+
+        int uncompressedSize = managedCursorInfo.getSerializedSize();
+        if (uncompressedSize > cursorInfoCompressionConfig.getCompressSizeThresholdInBytes()) {
+            MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata
+                    .newBuilder()
+                    .setCompressionType(compressionType)
+                    .setUncompressedSize(uncompressedSize)
+                    .build();
+            return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(),
+                    metadata.getSerializedSize(), compressionType);
+        }
+
+        return managedCursorInfo.toByteArray();
     }
 
     public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProtocolBufferException {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java
index 08d8fd939a01d..70ba4b543ec09 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java
@@ -19,11 +19,13 @@
 package org.apache.bookkeeper.mledger.impl;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.expectThrows;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.pulsar.common.api.proto.CompressionType;
 import org.testng.annotations.DataProvider;
@@ -49,16 +51,14 @@ private Object[][] compressionTypeProvider() {
         };
     }
 
-    @Test(dataProvider = "compressionTypeProvider")
-    public void testEncodeAndDecode(String compressionType) throws IOException {
-        long ledgerId = 10000;
+    private MLDataFormats.ManagedCursorInfo.Builder generateManagedCursorInfo(long ledgerId, int positionNumber) {
         MLDataFormats.ManagedCursorInfo.Builder builder = MLDataFormats.ManagedCursorInfo.newBuilder();
 
         builder.setCursorsLedgerId(ledgerId);
         builder.setMarkDeleteLedgerId(ledgerId);
 
         List batchedEntryDeletionIndexInfos = new ArrayList<>();
-        for (int i = 0; i < 1000; i++) {
+        for (int i = 0; i < positionNumber; i++) {
             MLDataFormats.NestedPositionInfo nestedPositionInfo = MLDataFormats.NestedPositionInfo.newBuilder()
                     .setEntryId(i).setLedgerId(i).build();
             MLDataFormats.BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = MLDataFormats
@@ -67,17 +67,24 @@ public void testEncodeAndDecode(String compressionType) throws IOException {
         }
         builder.addAllBatchedEntryDeletionIndexInfo(batchedEntryDeletionIndexInfos);
 
+        return builder;
+    }
+
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testEncodeAndDecode(String compressionType) throws IOException {
+        long ledgerId = 10000;
+        MLDataFormats.ManagedCursorInfo.Builder builder = generateManagedCursorInfo(ledgerId, 1000);
         MetaStoreImpl metaStore;
         if (INVALID_TYPE.equals(compressionType)) {
             IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> {
-                new MetaStoreImpl(null, null, null, compressionType);
+                new MetaStoreImpl(null, null, null, new MetadataCompressionConfig(compressionType));
             });
             assertEquals(compressionTypeEx.getMessage(),
                     "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType."
                             + compressionType);
             return;
         } else {
-            metaStore = new MetaStoreImpl(null, null, null, compressionType);
+            metaStore = new MetaStoreImpl(null, null, null, new MetadataCompressionConfig(compressionType));
         }
 
         MLDataFormats.ManagedCursorInfo managedCursorInfo = builder.build();
@@ -93,4 +100,42 @@ public void testEncodeAndDecode(String compressionType) throws IOException {
         MLDataFormats.ManagedCursorInfo info2 = metaStore.parseManagedCursorInfo(managedCursorInfo.toByteArray());
         assertEquals(info1, info2);
     }
+
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testCompressionThreshold(String compressionType) throws IOException {
+        int compressThreshold = 512;
+
+        long ledgerId = 10000;
+        // should not compress
+        MLDataFormats.ManagedCursorInfo smallInfo = generateManagedCursorInfo(ledgerId, 1).build();
+        assertTrue(smallInfo.getSerializedSize() < compressThreshold);
+
+        // should compress
+        MLDataFormats.ManagedCursorInfo bigInfo = generateManagedCursorInfo(ledgerId, 1000).build();
+        assertTrue(bigInfo.getSerializedSize() > compressThreshold);
+
+        MetaStoreImpl metaStore;
+        if (INVALID_TYPE.equals(compressionType)) {
+            IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> {
+                new MetaStoreImpl(null, null, null,
+                        new MetadataCompressionConfig(compressionType, compressThreshold));
+            });
+            assertEquals(compressionTypeEx.getMessage(),
+                    "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType."
+                            + compressionType);
+            return;
+        } else {
+            metaStore = new MetaStoreImpl(null, null, null,
+                    new MetadataCompressionConfig(compressionType, compressThreshold));
+        }
+
+        byte[] compressionBytes = metaStore.compressCursorInfo(smallInfo);
+        // not compressed
+        assertEquals(compressionBytes.length, smallInfo.getSerializedSize());
+
+
+        byte[] compressionBigBytes = metaStore.compressCursorInfo(bigInfo);
+        // compressed
+        assertTrue(compressionBigBytes.length != smallInfo.getSerializedSize());
+    }
 }
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java
index 7ddf6541c9a39..6e1f447225e53 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
 import org.apache.bookkeeper.mledger.offload.OffloadUtils;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.commons.lang3.RandomUtils;
@@ -33,6 +34,8 @@
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 
 /**
  * ManagedLedgerInfo metadata test.
@@ -53,11 +56,9 @@ private Object[][] compressionTypeProvider() {
         };
     }
 
-    @Test(dataProvider = "compressionTypeProvider")
-    public void testEncodeAndDecode(String compressionType) throws IOException {
-        long ledgerId = 10000;
+    private MLDataFormats.ManagedLedgerInfo.Builder generateManagedLedgerInfo(long ledgerId, int ledgerInfoNumber) {
         List ledgerInfoList = new ArrayList<>();
-        for (int i = 0; i < 100; i++) {
+        for (int i = 0; i < ledgerInfoNumber; i++) {
             MLDataFormats.ManagedLedgerInfo.LedgerInfo.Builder builder = MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder();
             builder.setLedgerId(ledgerId);
             builder.setEntries(RandomUtils.nextInt());
@@ -84,13 +85,18 @@ public void testEncodeAndDecode(String compressionType) throws IOException {
             ledgerId ++;
         }
 
-        MLDataFormats.ManagedLedgerInfo managedLedgerInfo = MLDataFormats.ManagedLedgerInfo.newBuilder()
-                .addAllLedgerInfo(ledgerInfoList)
-                .build();
+        return MLDataFormats.ManagedLedgerInfo.newBuilder()
+                .addAllLedgerInfo(ledgerInfoList);
+    }
+
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testEncodeAndDecode(String compressionType) throws IOException {
+        long ledgerId = 10000;
+        MLDataFormats.ManagedLedgerInfo managedLedgerInfo = generateManagedLedgerInfo(ledgerId,100).build();
 
         MetaStoreImpl metaStore;
         try {
-            metaStore = new MetaStoreImpl(null, null, compressionType, null);
+            metaStore = new MetaStoreImpl(null, null, new MetadataCompressionConfig(compressionType), null);
             if ("INVALID_TYPE".equals(compressionType)) {
                 Assert.fail("The managedLedgerInfo compression type is invalid, should fail.");
             }
@@ -126,4 +132,45 @@ public void testParseEmptyData() throws InvalidProtocolBufferException {
         Assert.assertEquals(managedLedgerInfo.toString(), "");
     }
 
+    @Test(dataProvider = "compressionTypeProvider")
+    public void testCompressionThreshold(String compressionType) {
+        long ledgerId = 10000;
+        int compressThreshold = 512;
+
+        // should not compress
+        MLDataFormats.ManagedLedgerInfo smallInfo = generateManagedLedgerInfo(ledgerId, 0).build();
+        assertTrue(smallInfo.getSerializedSize() < compressThreshold);
+
+        // should compress
+        MLDataFormats.ManagedLedgerInfo bigInfo = generateManagedLedgerInfo(ledgerId, 1000).build();
+        assertTrue(bigInfo.getSerializedSize() > compressThreshold);
+
+        MLDataFormats.ManagedLedgerInfo managedLedgerInfo = generateManagedLedgerInfo(ledgerId,100).build();
+
+        MetaStoreImpl metaStore;
+        try {
+            MetadataCompressionConfig metadataCompressionConfig =
+                    new MetadataCompressionConfig(compressionType, compressThreshold);
+            metaStore = new MetaStoreImpl(null, null, metadataCompressionConfig, null);
+            if ("INVALID_TYPE".equals(compressionType)) {
+                Assert.fail("The managedLedgerInfo compression type is invalid, should fail.");
+            }
+        } catch (Exception e) {
+            if ("INVALID_TYPE".equals(compressionType)) {
+                Assert.assertEquals(e.getClass(), IllegalArgumentException.class);
+                Assert.assertEquals(
+                        "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType."
+                                + compressionType, e.getMessage());
+                return;
+            } else {
+                throw e;
+            }
+        }
+
+        byte[] compressionBytes = metaStore.compressLedgerInfo(smallInfo);
+        assertEquals(compressionBytes.length, smallInfo.getSerializedSize());
+
+        byte[] compressionBytesBig = metaStore.compressLedgerInfo(bigInfo);
+        assertTrue(compressionBytesBig.length !=smallInfo.getSerializedSize());
+    }
 }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 5dc15bd6a4dcf..c9bb4b86ba780 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2135,12 +2135,25 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "If value is invalid or NONE, then save the ManagedLedgerInfo bytes data directly.")
     private String managedLedgerInfoCompressionType = "NONE";
 
+    @FieldContext(category = CATEGORY_STORAGE_ML,
+            doc = "ManagedLedgerInfo compression size threshold (bytes), "
+                    + "only compress metadata when origin size more then this value.\n"
+                    + "0 means compression will always apply.\n")
+    private long managedLedgerInfoCompressionThresholdInBytes = 16 * 1024;
+
 
     @FieldContext(category = CATEGORY_STORAGE_ML,
             doc = "ManagedCursorInfo compression type, option values (NONE, LZ4, ZLIB, ZSTD, SNAPPY). \n"
                     + "If value is NONE, then save the ManagedCursorInfo bytes data directly.")
     private String managedCursorInfoCompressionType = "NONE";
 
+
+    @FieldContext(category = CATEGORY_STORAGE_ML,
+            doc = "ManagedCursorInfo compression size threshold (bytes), "
+                    + "only compress metadata when origin size more then this value.\n"
+                    + "0 means compression will always apply.\n")
+    private long managedCursorInfoCompressionThresholdInBytes = 16 * 1024;
+
     @FieldContext(
             dynamic = true,
             category = CATEGORY_STORAGE_ML,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
index b16b9a7dd4833..d86649abd3c93 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
@@ -70,8 +70,12 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata
         managedLedgerFactoryConfig.setTraceTaskExecution(conf.isManagedLedgerTraceTaskExecution());
         managedLedgerFactoryConfig.setCursorPositionFlushSeconds(conf.getManagedLedgerCursorPositionFlushSeconds());
         managedLedgerFactoryConfig.setManagedLedgerInfoCompressionType(conf.getManagedLedgerInfoCompressionType());
+        managedLedgerFactoryConfig.setManagedLedgerInfoCompressionThresholdInBytes(
+                conf.getManagedLedgerInfoCompressionThresholdInBytes());
         managedLedgerFactoryConfig.setStatsPeriodSeconds(conf.getManagedLedgerStatsPeriodSeconds());
         managedLedgerFactoryConfig.setManagedCursorInfoCompressionType(conf.getManagedCursorInfoCompressionType());
+        managedLedgerFactoryConfig.setManagedCursorInfoCompressionThresholdInBytes(
+                conf.getManagedCursorInfoCompressionThresholdInBytes());
 
         Configuration configuration = new ClientConfiguration();
         if (conf.isBookkeeperClientExposeStatsToPrometheus()) {

From da2a1482a7dbd57bfc7c130315db249fe53e0260 Mon Sep 17 00:00:00 2001
From: tison 
Date: Wed, 31 May 2023 09:49:18 +0800
Subject: [PATCH 345/404] [fix][cli] Fulfill add-opens to function-localrunner
 also (#20417)

Signed-off-by: tison 
---
 bin/bookkeeper             |  2 +-
 bin/function-localrunner   | 14 ++++++++------
 bin/pulsar                 |  2 +-
 bin/pulsar-admin-common.sh |  2 +-
 bin/pulsar-perf            |  2 +-
 5 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/bin/bookkeeper b/bin/bookkeeper
index fb516a98acdc2..0cc07dd49aba5 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -168,7 +168,7 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $BOOKIE_LOG_CONF`"
 # Allow Netty to use reflection access
 OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
 
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then
diff --git a/bin/function-localrunner b/bin/function-localrunner
index 45a37cb306794..2e0aa0f6dffe2 100755
--- a/bin/function-localrunner
+++ b/bin/function-localrunner
@@ -40,13 +40,15 @@ PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
 PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}
 
 # Garbage collection log.
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
-# java version has space, use [[ -n $PARAM ]] to judge if variable exists
-if [[ -n "$IS_JAVA_8" ]]; then
-  PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"}
-else
-# After jdk 9, gc log param should config like this. Ignoring version less than jdk 8
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
+if [[ -z "$IS_JAVA_8" ]]; then
+  # >= JDK 9
   PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xlog:gc:logs/pulsar_gc_%p.log:time,uptime:filecount=10,filesize=20M"}
+  # '--add-opens' option is not supported in JDK 1.8
+  OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED"
+else
+  # == JDK 1.8
+  PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"}
 fi
 
 # Extra options to be passed to the jvm
diff --git a/bin/pulsar b/bin/pulsar
index e3b22caced52e..20ed1f7f22b0f 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -291,7 +291,7 @@ OPTS="$OPTS -Dzookeeper.clientTcpKeepAlive=true"
 
 # Allow Netty to use reflection access
 OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then
diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh
index 8223ac5b3bf24..8aa21c00f634d 100755
--- a/bin/pulsar-admin-common.sh
+++ b/bin/pulsar-admin-common.sh
@@ -91,7 +91,7 @@ PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
 OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
 
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then
diff --git a/bin/pulsar-perf b/bin/pulsar-perf
index 47c02bc3d67d5..bdc1dc1ed8b8c 100755
--- a/bin/pulsar-perf
+++ b/bin/pulsar-perf
@@ -134,7 +134,7 @@ PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
 OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true"
 
-IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'`
+IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
 # Start --add-opens options
 # '--add-opens' option is not supported in jdk8
 if [[ -z "$IS_JAVA_8" ]]; then

From f92011762deea79040fb006fab0d0f297fae9e1f Mon Sep 17 00:00:00 2001
From: ken <1647023764@qq.com>
Date: Wed, 31 May 2023 10:10:00 +0800
Subject: [PATCH 346/404] [improve][broker] Change limitStatsLogging config
 default value to true (#20409)

Co-authored-by: fanjianye 
---
 conf/broker.conf                                   |  4 ++--
 conf/standalone.conf                               |  4 ++--
 deployment/terraform-ansible/templates/broker.conf |  4 ++--
 .../apache/pulsar/broker/ServiceConfiguration.java |  2 +-
 .../broker/BookKeeperClientFactoryImplTest.java    | 14 +++++++-------
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index ca118d254f496..22ca71864e9cd 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1078,8 +1078,8 @@ bookkeeperExplicitLacIntervalInMills=0
 # bookkeeperClientExposeStatsToPrometheus=false
 
 # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
-# to limit per_channel_bookie_client metrics. default is false
-# bookkeeperClientLimitStatsLogging=false
+# to limit per_channel_bookie_client metrics. default is true
+# bookkeeperClientLimitStatsLogging=true
 
 ### --- Managed Ledger --- ###
 
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 8d24d5ad88c7f..46e6aed76e42a 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -697,8 +697,8 @@ bookkeeperUseV2WireProtocol=true
 # bookkeeperClientExposeStatsToPrometheus=false
 
 # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
-# to limit per_channel_bookie_client metrics. default is false
-# bookkeeperClientLimitStatsLogging=false
+# to limit per_channel_bookie_client metrics. default is true
+# bookkeeperClientLimitStatsLogging=true
 
 ### --- Managed Ledger --- ###
 
diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
index f42d4c807d5d9..37e512fb35cc6 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -745,8 +745,8 @@ bookkeeperExplicitLacIntervalInMills=0
 # bookkeeperClientExposeStatsToPrometheus=false
 
 # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
-# to limit per_channel_bookie_client metrics. default is false
-# bookkeeperClientLimitStatsLogging=false
+# to limit per_channel_bookie_client metrics. default is true
+# bookkeeperClientLimitStatsLogging=true
 
 ### --- Managed Ledger --- ###
 
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index c9bb4b86ba780..a709e49e3a980 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1820,7 +1820,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
             category = CATEGORY_STORAGE_BK,
             doc = "whether limit per_channel_bookie_client metrics of bookkeeper client stats"
     )
-    private boolean bookkeeperClientLimitStatsLogging = false;
+    private boolean bookkeeperClientLimitStatsLogging = true;
 
     @FieldContext(
             category = CATEGORY_STORAGE_BK,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
index a02689dc9763a..0dea84e727a88 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
@@ -303,22 +303,22 @@ public void testBookKeeperIoThreadsConfiguration() throws Exception {
     public void testBookKeeperLimitStatsLoggingConfiguration() throws Exception {
         BookKeeperClientFactoryImpl factory = new BookKeeperClientFactoryImpl();
         ServiceConfiguration conf = new ServiceConfiguration();
-        assertFalse(
-                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging());
+        assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)
+                .getLimitStatsLogging());
         EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class);
         BookKeeper.Builder builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class),
                 factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf));
         ClientConfiguration clientConfiguration =
                 (ClientConfiguration) FieldUtils.readField(builder, "conf", true);
-        assertFalse(clientConfiguration.getLimitStatsLogging());
+        assertTrue(clientConfiguration.getLimitStatsLogging());
 
-        conf.setBookkeeperClientLimitStatsLogging(true);
-        assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)
-                .getLimitStatsLogging());
+        conf.setBookkeeperClientLimitStatsLogging(false);
+        assertFalse(
+                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging());
         builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class),
                 factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf));
         clientConfiguration =
                 (ClientConfiguration) FieldUtils.readField(builder, "conf", true);
-        assertTrue(clientConfiguration.getLimitStatsLogging());
+        assertFalse(clientConfiguration.getLimitStatsLogging());
     }
 }

From 563a573168ba51b60f060a993416d7ea642cc27b Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Wed, 31 May 2023 10:18:03 +0800
Subject: [PATCH 347/404] [fix][fn] Fix function update error (#19895)

---
 .../worker/rest/api/FunctionsImpl.java        |  5 +++-
 .../functions/worker/rest/api/SinksImpl.java  |  5 +++-
 .../worker/rest/api/SourcesImpl.java          |  5 +++-
 .../functions/PulsarFunctionsTest.java        | 29 +++++++++++++++++++
 4 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index c7967600da5f6..715d660ddff0d 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -55,6 +55,7 @@
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.utils.ComponentTypeUtils;
 import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
 import org.apache.pulsar.functions.utils.functions.FunctionArchive;
 import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
 import org.apache.pulsar.functions.worker.FunctionsManager;
@@ -374,8 +375,10 @@ public void updateFunction(final String tenant,
 
             Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder;
             if (isNotBlank(functionPkgUrl) || uploadedInputStream != null) {
+                Function.FunctionMetaData metaData = functionMetaDataBuilder.build();
+                metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData);
                 try {
-                    packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(),
+                    packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData,
                             functionPkgUrl, fileDetail, componentPackageFile);
                 } catch (Exception e) {
                     log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType),
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
index 98450c4a0b5d0..5370fe93a7a30 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.utils.ComponentTypeUtils;
+import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
 import org.apache.pulsar.functions.utils.SinkConfigUtils;
 import org.apache.pulsar.functions.utils.io.Connector;
 import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
@@ -378,8 +379,10 @@ public void updateSink(final String tenant,
 
             Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder;
             if (isNotBlank(sinkPkgUrl) || uploadedInputStream != null) {
+                Function.FunctionMetaData metaData = functionMetaDataBuilder.build();
+                metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData);
                 try {
-                    packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(),
+                    packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData,
                             sinkPkgUrl, fileDetail, componentPackageFile);
                 } catch (Exception e) {
                     log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType),
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
index 876c7e7572e78..2f491424d658a 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
@@ -53,6 +53,7 @@
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.InstanceCommunication;
 import org.apache.pulsar.functions.utils.ComponentTypeUtils;
+import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
 import org.apache.pulsar.functions.utils.SourceConfigUtils;
 import org.apache.pulsar.functions.utils.io.Connector;
 import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
@@ -372,8 +373,10 @@ public void updateSource(final String tenant,
 
             Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder;
             if (isNotBlank(sourcePkgUrl) || uploadedInputStream != null) {
+                Function.FunctionMetaData metaData = functionMetaDataBuilder.build();
+                metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData);
                 try {
-                    packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(),
+                    packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData,
                             sourcePkgUrl, fileDetail, componentPackageFile);
                 } catch (Exception e) {
                     log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType),
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
index 1e54764ad5d2b..e6ba6acff83c0 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java
@@ -729,6 +729,19 @@ protected void testExclamationFunction(Runtime runtime,
         //get function status
         getFunctionStatus(functionName, 0, true, 2);
 
+        // update code file
+        switch (runtime) {
+            case JAVA:
+                updateFunctionCodeFile(functionName, Runtime.JAVA, "test");
+                break;
+            case PYTHON:
+                updateFunctionCodeFile(functionName, Runtime.PYTHON, EXCLAMATION_PYTHON_FILE);
+                break;
+            case GO:
+                updateFunctionCodeFile(functionName, Runtime.GO, EXCLAMATION_GO_FILE);
+                break;
+        }
+
         // delete function
         deleteFunction(functionName);
 
@@ -894,6 +907,22 @@ private void updateFunctionParallelism(String functionName, int parallelism) thr
         assertTrue(result.getStdout().contains("Updated successfully"));
     }
 
+    private void updateFunctionCodeFile(String functionName, Runtime runtime, String codeFile) throws Exception {
+
+        CommandGenerator generator = new CommandGenerator();
+        generator.setFunctionName(functionName);
+        generator.setRuntime(runtime);
+        String command = generator.generateUpdateFunctionCommand(codeFile);
+
+        log.info("---------- Function command: {}", command);
+        String[] commands = {
+                "sh", "-c", command
+        };
+        ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(
+                commands);
+        assertTrue(result.getStdout().contains("Updated successfully"));
+    }
+
     protected  void submitFunction(Runtime runtime,
                                       String inputTopicName,
                                       String outputTopicName,

From fb1b46e5e993d77c583f715d2bea2eadbb052a81 Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Tue, 30 May 2023 23:03:36 -0400
Subject: [PATCH 348/404] [fix][fn] Go functions must retrieve consumers by
 non-particioned topic ID (#20413)

Co-authored-by: Andy Walker 
---
 pulsar-function-go/pf/instance.go | 18 ++++++++++++++++--
 pulsar-function-go/pf/util.go     | 11 +++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 138489444d160..6f73f3e631212 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -404,11 +404,25 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) {
 // ackInputMessage doesn't produce any result, or the user doesn't want the result.
 func (gi *goInstance) ackInputMessage(inputMessage pulsar.Message) {
 	log.Debugf("ack input message topic name is: %s", inputMessage.Topic())
-	gi.consumers[inputMessage.Topic()].Ack(inputMessage)
+	gi.respondMessage(inputMessage, true)
 }
 
 func (gi *goInstance) nackInputMessage(inputMessage pulsar.Message) {
-	gi.consumers[inputMessage.Topic()].Nack(inputMessage)
+	gi.respondMessage(inputMessage, false)
+}
+
+func (gi *goInstance) respondMessage(inputMessage pulsar.Message, ack bool) {
+	topicName, err := ParseTopicName(inputMessage.Topic())
+	if err != nil {
+		log.Errorf("unable respond to message ID %s - invalid topic: %v", messageIDStr(inputMessage), err)
+		return
+	}
+	// consumers are indexed by topic name only (no partition)
+	if ack {
+		gi.consumers[topicName.NameWithoutPartition()].Ack(inputMessage)
+		return
+	}
+	gi.consumers[topicName.NameWithoutPartition()].Nack(inputMessage)
 }
 
 func getIdleTimeout(timeoutMilliSecond time.Duration) time.Duration {
diff --git a/pulsar-function-go/pf/util.go b/pulsar-function-go/pf/util.go
index d5b32da841121..1d1aa1cab939f 100644
--- a/pulsar-function-go/pf/util.go
+++ b/pulsar-function-go/pf/util.go
@@ -21,6 +21,8 @@ package pf
 
 import (
 	"fmt"
+
+	"github.com/apache/pulsar-client-go/pulsar"
 )
 
 func getProperties(fullyQualifiedName string, instanceID int) map[string]string {
@@ -39,3 +41,12 @@ func getDefaultSubscriptionName(tenant, namespace, name string) string {
 func getFullyQualifiedInstanceID(tenant, namespace, name string, instanceID int) string {
 	return fmt.Sprintf("%s/%s/%s:%d", tenant, namespace, name, instanceID)
 }
+
+func messageIDStr(msg pulsar.Message) string {
+	// :::
+	return fmt.Sprintf("%d:%d:%d:%d",
+		msg.ID().LedgerID(),
+		msg.ID().EntryID(),
+		msg.ID().PartitionIdx(),
+		msg.ID().BatchIdx())
+}

From 5c74d207426e26ec4feea279ed2ae1b1c2909cba Mon Sep 17 00:00:00 2001
From: thetumbled <52550727+thetumbled@users.noreply.github.com>
Date: Wed, 31 May 2023 14:34:32 +0800
Subject: [PATCH 349/404] [fix] [broker] do not filter system topic while
 shedding. (#18949)

### Motivation

Currently, topics/bundles in `pulsar/system` will be filter while doing shedding, which is introduced by mistake by pr https://github.com/apache/pulsar/pull/15252.
But we need to unload topics/bundles in `pulsar/system` for load balancing.

### Modifications

do not filter topics/bundles in `pulsar/system`.
---
 .../apache/pulsar/broker/loadbalance/LoadData.java    |  2 +-
 .../pulsar/broker/namespace/NamespaceService.java     | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
index 87f630f1a09fb..a632a47f05116 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java
@@ -64,7 +64,7 @@ public Map getBundleData() {
 
     public Map getBundleDataForLoadShedding() {
         return bundleData.entrySet().stream()
-                .filter(e -> !NamespaceService.isSystemServiceNamespace(
+                .filter(e -> !NamespaceService.filterNamespaceForShedding(
                         NamespaceBundle.getBundleNamespace(e.getKey())))
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 9d8d9e3890a19..e2d4ef5153769 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1607,6 +1607,17 @@ public static boolean isSystemServiceNamespace(String namespace) {
                 || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches();
     }
 
+    /**
+     * used for filtering bundles in special namespace.
+     * @param namespace
+     * @return True if namespace is HEARTBEAT_NAMESPACE or SLA_NAMESPACE
+     */
+    public static boolean filterNamespaceForShedding(String namespace) {
+        return SLA_NAMESPACE_PATTERN.matcher(namespace).matches()
+                || HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches()
+                || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches();
+    }
+
     public static boolean isHeartbeatNamespace(ServiceUnitId ns) {
         String namespace = ns.getNamespaceObject().toString();
         return HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches()

From 8e0ebba345f5f02bf2434fec9a75268d92b1b60d Mon Sep 17 00:00:00 2001
From: YingQun Zhong 
Date: Wed, 31 May 2023 15:58:51 +0800
Subject: [PATCH 350/404] [fix][offload] fix offload metrics error (#20366)

---
 .../pulsar/common/naming/TopicName.java       | 36 +++++++++++++++++++
 .../pulsar/common/naming/TopicNameTest.java   | 34 ++++++++++++++++++
 .../impl/FileStoreBackedReadHandleImpl.java   | 11 +++---
 .../FileSystemManagedLedgerOffloader.java     | 22 ++++++++----
 .../FileSystemManagedLedgerOffloaderTest.java | 24 +++++++------
 .../impl/BlobStoreBackedInputStreamImpl.java  |  9 +++--
 .../impl/BlobStoreBackedReadHandleImpl.java   |  4 ++-
 .../impl/BlobStoreBackedReadHandleImplV2.java |  4 ++-
 .../impl/BlobStoreManagedLedgerOffloader.java | 12 ++++---
 .../BlockAwareSegmentInputStreamImpl.java     |  7 ++--
 .../BlobStoreManagedLedgerOffloaderTest.java  |  9 ++---
 11 files changed, 135 insertions(+), 37 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
index 79ef64c1ae459..eebca0e0d7214 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java
@@ -339,6 +339,42 @@ public String getPersistenceNamingEncoding() {
         }
     }
 
+    /**
+     * get topic full name from managedLedgerName.
+     *
+     * @return the topic full name, format -> domain://tenant/namespace/topic
+     */
+    public static String fromPersistenceNamingEncoding(String mlName) {
+        // The managedLedgerName convention is: tenant/namespace/domain/topic
+        // We want to transform to topic full name in the order: domain://tenant/namespace/topic
+        if (mlName == null || mlName.length() == 0) {
+            return mlName;
+        }
+        List parts = Splitter.on("/").splitToList(mlName);
+        String tenant;
+        String cluster;
+        String namespacePortion;
+        String domain;
+        String localName;
+        if (parts.size() == 4) {
+            tenant = parts.get(0);
+            cluster = null;
+            namespacePortion = parts.get(1);
+            domain = parts.get(2);
+            localName = parts.get(3);
+            return String.format("%s://%s/%s/%s", domain, tenant, namespacePortion, localName);
+        } else if (parts.size() == 5) {
+            tenant = parts.get(0);
+            cluster = parts.get(1);
+            namespacePortion = parts.get(2);
+            domain = parts.get(3);
+            localName = parts.get(4);
+            return String.format("%s://%s/%s/%s/%s", domain, tenant, cluster, namespacePortion, localName);
+        } else {
+            throw new IllegalArgumentException("Invalid managedLedger name: " + mlName);
+        }
+    }
+
     /**
      * Get a string suitable for completeTopicName lookup.
      *
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
index 8e32fbe3d33c0..835045f9167dd 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java
@@ -236,6 +236,40 @@ public void testDecodeEncode() throws Exception {
         assertEquals(name.getPersistenceNamingEncoding(), "prop/colo/ns/persistent/" + encodedName);
     }
 
+    @Test
+    public void testFromPersistenceNamingEncoding() {
+        // case1: V2
+        String mlName1 = "public_tenant/default_namespace/persistent/test_topic";
+        String expectedTopicName1 = "persistent://public_tenant/default_namespace/test_topic";
+
+        TopicName name1 = TopicName.get(expectedTopicName1);
+        assertEquals(name1.getPersistenceNamingEncoding(), mlName1);
+        assertEquals(TopicName.fromPersistenceNamingEncoding(mlName1), expectedTopicName1);
+
+        // case2: V1
+        String mlName2 = "public_tenant/my_cluster/default_namespace/persistent/test_topic";
+        String expectedTopicName2 = "persistent://public_tenant/my_cluster/default_namespace/test_topic";
+
+        TopicName name2 = TopicName.get(expectedTopicName2);
+        assertEquals(name2.getPersistenceNamingEncoding(), mlName2);
+        assertEquals(TopicName.fromPersistenceNamingEncoding(mlName2), expectedTopicName2);
+
+        // case3: null
+        String mlName3 = "";
+        String expectedTopicName3 = "";
+        assertEquals(expectedTopicName3, TopicName.fromPersistenceNamingEncoding(mlName3));
+
+        // case4: Invalid name
+        try {
+            String mlName4 = "public_tenant/my_cluster/default_namespace/persistent/test_topic/sub_topic";
+            TopicName.fromPersistenceNamingEncoding(mlName4);
+            fail("Should have raised exception");
+        } catch (IllegalArgumentException e) {
+            // Exception is expected.
+        }
+    }
+
+
     @SuppressWarnings("deprecation")
     @Test
     public void testTopicNameWithoutCluster() throws Exception {
diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java
index bdeb88e9ac93c..506fbb8de68bf 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.MapFile;
+import org.apache.pulsar.common.naming.TopicName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,6 +52,7 @@ public class FileStoreBackedReadHandleImpl implements ReadHandle {
     private final LedgerMetadata ledgerMetadata;
     private final LedgerOffloaderStats offloaderStats;
     private final String managedLedgerName;
+    private final String topicName;
 
     private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader reader, long ledgerId,
                                           LedgerOffloaderStats offloaderStats,
@@ -60,13 +62,14 @@ private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader r
         this.reader = reader;
         this.offloaderStats = offloaderStats;
         this.managedLedgerName = managedLedgerName;
+        this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         LongWritable key = new LongWritable();
         BytesWritable value = new BytesWritable();
         try {
             key.set(FileSystemManagedLedgerOffloader.METADATA_KEY_INDEX);
             long startReadIndexTime = System.nanoTime();
             reader.get(key, value);
-            offloaderStats.recordReadOffloadIndexLatency(managedLedgerName,
+            offloaderStats.recordReadOffloadIndexLatency(topicName,
                     System.nanoTime() - startReadIndexTime, TimeUnit.NANOSECONDS);
             this.ledgerMetadata = parseLedgerMetadata(ledgerId, value.copyBytes());
         } catch (IOException e) {
@@ -125,7 +128,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr
                 while (entriesToRead > 0) {
                     long startReadTime = System.nanoTime();
                     reader.next(key, value);
-                    this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName,
+                    this.offloaderStats.recordReadOffloadDataLatency(topicName,
                             System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS);
                     int length = value.getLength();
                     long entryId = key.get();
@@ -135,7 +138,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr
                         buf.writeBytes(value.copyBytes());
                         entriesToRead--;
                         nextExpectedId++;
-                        this.offloaderStats.recordReadOffloadBytes(managedLedgerName, length);
+                        this.offloaderStats.recordReadOffloadBytes(topicName, length);
                     } else if (entryId > lastEntry) {
                         log.info("Expected to read {}, but read {}, which is greater than last entry {}",
                                 nextExpectedId, entryId, lastEntry);
@@ -144,7 +147,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr
             }
                 promise.complete(LedgerEntriesImpl.create(entries));
             } catch (Throwable t) {
-                this.offloaderStats.recordReadOffloadError(managedLedgerName);
+                this.offloaderStats.recordReadOffloadError(topicName);
                 promise.completeExceptionally(t);
                 entries.forEach(LedgerEntry::close);
             }
diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
index d5e09ba725421..25b63374946c8 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.MapFile;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -197,9 +198,10 @@ public void run() {
                 return;
             }
             long ledgerId = readHandle.getId();
-            final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME);
-            String storagePath = getStoragePath(storageBasePath, topicName);
+            final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME);
+            String storagePath = getStoragePath(storageBasePath, managedLedgerName);
             String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid);
+            final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
             LongWritable key = new LongWritable();
             BytesWritable value = new BytesWritable();
             try {
@@ -241,7 +243,7 @@ public void run() {
                 promise.complete(null);
             } catch (Exception e) {
                 log.error("Exception when get CompletableFuture : ManagerLedgerName: {}, "
-                        + "LedgerId: {}, UUID: {} ", topicName, ledgerId, uuid, e);
+                        + "LedgerId: {}, UUID: {} ", managedLedgerName, ledgerId, uuid, e);
                 if (e instanceof InterruptedException) {
                     Thread.currentThread().interrupt();
                 }
@@ -306,22 +308,27 @@ public static FileSystemWriter create(LedgerEntries ledgerEntriesOnce,
         @Override
         public void run() {
             String managedLedgerName = ledgerReader.extraMetadata.get(MANAGED_LEDGER_NAME);
+            String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
             if (ledgerReader.fileSystemWriteException == null) {
                 Iterator iterator = ledgerEntriesOnce.iterator();
                 while (iterator.hasNext()) {
                     LedgerEntry entry = iterator.next();
                     long entryId = entry.getEntryId();
                     key.set(entryId);
+                    byte[] currentEntryBytes;
+                    int currentEntrySize;
                     try {
-                        value.set(entry.getEntryBytes(), 0, entry.getEntryBytes().length);
+                        currentEntryBytes = entry.getEntryBytes();
+                        currentEntrySize = currentEntryBytes.length;
+                        value.set(currentEntryBytes, 0, currentEntrySize);
                         dataWriter.append(key, value);
                     } catch (IOException e) {
                         ledgerReader.fileSystemWriteException = e;
-                        ledgerReader.offloaderStats.recordWriteToStorageError(managedLedgerName);
+                        ledgerReader.offloaderStats.recordWriteToStorageError(topicName);
                         break;
                     }
                     haveOffloadEntryNumber.incrementAndGet();
-                    ledgerReader.offloaderStats.recordOffloadBytes(managedLedgerName, entry.getLength());
+                    ledgerReader.offloaderStats.recordOffloadBytes(topicName, currentEntrySize);
                 }
             }
             countDownLatch.countDown();
@@ -367,6 +374,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map promise = new CompletableFuture<>();
         try {
             fileSystem.delete(new Path(dataFilePath), true);
@@ -376,7 +384,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map
-                this.offloaderStats.recordDeleteOffloadOps(ledgerName, t == null));
+                this.offloaderStats.recordDeleteOffloadOps(topicName, t == null));
     }
 
     @Override
diff --git a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java
index 1aebab571c971..b9de5d1a49e9a 100644
--- a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java
+++ b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.pulsar.common.naming.TopicName;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 import java.net.URI;
@@ -46,8 +47,9 @@
 
 public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase {
     private final PulsarMockBookKeeper bk;
-    private String topic = "public/default/testOffload";
-    private String storagePath = createStoragePath(topic);
+    private String managedLedgerName = "public/default/persistent/testOffload";
+    private String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
+    private String storagePath = createStoragePath(managedLedgerName);
     private LedgerHandle lh;
     private ReadHandle toWrite;
     private final int numberOfEntries = 601;
@@ -56,7 +58,7 @@ public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase {
     public FileSystemManagedLedgerOffloaderTest() throws Exception {
         this.bk = new PulsarMockBookKeeper(scheduler);
         this.toWrite = buildReadHandle();
-        map.put("ManagedLedgerName", topic);
+        map.put("ManagedLedgerName", managedLedgerName);
     }
 
     private ReadHandle buildReadHandle() throws Exception {
@@ -125,10 +127,10 @@ public void testOffloadAndReadMetrics() throws Exception {
         offloader.offload(toWrite, uuid, map).get();
 
         LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats;
-        assertTrue(offloaderStats.getOffloadError(topic) == 0);
-        assertTrue(offloaderStats.getOffloadBytes(topic) > 0);
-        assertTrue(offloaderStats.getReadLedgerLatency(topic).count > 0);
-        assertTrue(offloaderStats.getWriteStorageError(topic) == 0);
+        assertTrue(offloaderStats.getOffloadError(topicName) == 0);
+        assertTrue(offloaderStats.getOffloadBytes(topicName) > 0);
+        assertTrue(offloaderStats.getReadLedgerLatency(topicName).count > 0);
+        assertTrue(offloaderStats.getWriteStorageError(topicName) == 0);
 
         ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get();
         LedgerEntries toTestEntries = toTest.read(0, numberOfEntries - 1);
@@ -137,10 +139,10 @@ public void testOffloadAndReadMetrics() throws Exception {
             LedgerEntry toTestEntry = toTestIter.next();
         }
 
-        assertTrue(offloaderStats.getReadOffloadError(topic) == 0);
-        assertTrue(offloaderStats.getReadOffloadBytes(topic) > 0);
-        assertTrue(offloaderStats.getReadOffloadDataLatency(topic).count > 0);
-        assertTrue(offloaderStats.getReadOffloadIndexLatency(topic).count > 0);
+        assertTrue(offloaderStats.getReadOffloadError(topicName) == 0);
+        assertTrue(offloaderStats.getReadOffloadBytes(topicName) > 0);
+        assertTrue(offloaderStats.getReadOffloadDataLatency(topicName).count > 0);
+        assertTrue(offloaderStats.getReadOffloadIndexLatency(topicName).count > 0);
     }
 
     @Test
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java
index aa27df46c5e65..0dea46726f50a 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java
@@ -26,6 +26,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.BackedInputStream;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.options.GetOptions;
@@ -44,6 +45,7 @@ public class BlobStoreBackedInputStreamImpl extends BackedInputStream {
     private final int bufferSize;
     private LedgerOffloaderStats offloaderStats;
     private String managedLedgerName;
+    private String topicName;
 
     private long cursor;
     private long bufferOffsetStart;
@@ -71,6 +73,7 @@ public BlobStoreBackedInputStreamImpl(BlobStore blobStore, String bucket, String
         this(blobStore, bucket, key, versionCheck, objectLen, bufferSize);
         this.offloaderStats = offloaderStats;
         this.managedLedgerName = managedLedgerName;
+        this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
     }
 
     /**
@@ -110,13 +113,13 @@ private boolean refillBufferIfNeeded() throws IOException {
                 // because JClouds streams the content
                 // and actually the HTTP call finishes when the stream is fully read
                 if (this.offloaderStats != null) {
-                    this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName,
+                    this.offloaderStats.recordReadOffloadDataLatency(topicName,
                             System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS);
-                    this.offloaderStats.recordReadOffloadBytes(managedLedgerName, endRange - startRange + 1);
+                    this.offloaderStats.recordReadOffloadBytes(topicName, endRange - startRange + 1);
                 }
             } catch (Throwable e) {
                 if (null != this.offloaderStats) {
-                    this.offloaderStats.recordReadOffloadError(this.managedLedgerName);
+                    this.offloaderStats.recordReadOffloadError(this.topicName);
                 }
                 throw new IOException("Error reading from BlobStore", e);
             }
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java
index cdabe5ece0ba2..5a571bb208e34 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java
@@ -45,6 +45,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockBuilder;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.slf4j.Logger;
@@ -261,6 +262,7 @@ public static ReadHandle open(ScheduledExecutorService executor,
         int retryCount = 3;
         OffloadIndexBlock index = null;
         IOException lastException = null;
+        String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         // The following retry is used to avoid to some network issue cause read index file failure.
         // If it can not recovery in the retry, we will throw the exception and the dispatcher will schedule to
         // next read.
@@ -269,7 +271,7 @@ public static ReadHandle open(ScheduledExecutorService executor,
         while (retryCount-- > 0) {
             long readIndexStartTime = System.nanoTime();
             Blob blob = blobStore.getBlob(bucket, indexKey);
-            offloaderStats.recordReadOffloadIndexLatency(managedLedgerName,
+            offloaderStats.recordReadOffloadIndexLatency(topicName,
                     System.nanoTime() - readIndexStartTime, TimeUnit.NANOSECONDS);
             versionCheck.check(indexKey, blob);
             OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create();
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java
index 2e3d0b08970ca..e40a0a3834c85 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java
@@ -46,6 +46,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockV2Builder;
 import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.slf4j.Logger;
@@ -297,13 +298,14 @@ public static ReadHandle open(ScheduledExecutorService executor,
             throws IOException {
         List inputStreams = new LinkedList<>();
         List indice = new LinkedList<>();
+        String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         for (int i = 0; i < indexKeys.size(); i++) {
             String indexKey = indexKeys.get(i);
             String key = keys.get(i);
             log.debug("open bucket: {} index key: {}", bucket, indexKey);
             long startTime = System.nanoTime();
             Blob blob = blobStore.getBlob(bucket, indexKey);
-            offloaderStats.recordReadOffloadIndexLatency(managedLedgerName,
+            offloaderStats.recordReadOffloadIndexLatency(topicName,
                     System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
             log.debug("indexKey blob: {} {}", indexKey, blob);
             versionCheck.check(indexKey, blob);
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
index 6d69b5edbc3fb..7a15c414aa8c4 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
@@ -63,6 +63,7 @@
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.BlobStoreLocation;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
@@ -176,7 +177,8 @@ public Map getOffloadDriverMetadata() {
     public CompletableFuture offload(ReadHandle readHandle,
                                            UUID uuid,
                                            Map extraMetadata) {
-        final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME);
+        final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME);
+        final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName);
         final BlobStore writeBlobStore = blobStores.get(config.getBlobStoreLocation());
         log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata,
                 config.getBlobStoreLocation(), writeBlobStore);
@@ -226,7 +228,7 @@ public CompletableFuture offload(ReadHandle readHandle,
                         .calculateBlockSize(config.getMaxBlockSizeInBytes(), readHandle, startEntry, entryBytesWritten);
 
                     try (BlockAwareSegmentInputStream blockStream = new BlockAwareSegmentInputStreamImpl(
-                            readHandle, startEntry, blockSize, this.offloaderStats, topicName)) {
+                            readHandle, startEntry, blockSize, this.offloaderStats, managedLedgerName)) {
 
                         Payload partPayload = Payloads.newInputStreamPayload(blockStream);
                         partPayload.getContentMetadata().setContentLength((long) blockSize);
@@ -611,7 +613,8 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid,
 
         return promise.whenComplete((__, t) -> {
             if (null != this.ml) {
-                this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null);
+                this.offloaderStats.recordDeleteOffloadOps(
+                  TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null);
             }
         });
     }
@@ -636,7 +639,8 @@ public CompletableFuture deleteOffloaded(UUID uid, Map off
         });
 
         return promise.whenComplete((__, t) ->
-                this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null));
+                this.offloaderStats.recordDeleteOffloadOps(
+                  TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null));
     }
 
     @Override
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java
index d07fbdb92477b..06d7f2129ba31 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java
@@ -36,6 +36,7 @@
 import org.apache.bookkeeper.mledger.LedgerOffloaderStats;
 import org.apache.bookkeeper.mledger.offload.jcloud.BlockAwareSegmentInputStream;
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.naming.TopicName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,6 +74,7 @@ public class BlockAwareSegmentInputStreamImpl extends BlockAwareSegmentInputStre
     // Keep a list of all entries ByteBuf, each ByteBuf contains 2 buf: entry header and entry content.
     private List entriesByteBuf = null;
     private LedgerOffloaderStats offloaderStats;
+    private String managedLedgerName;
     private String topicName;
     private int currentOffset = 0;
     private final AtomicBoolean close = new AtomicBoolean(false);
@@ -91,7 +93,8 @@ public BlockAwareSegmentInputStreamImpl(ReadHandle ledger, long startEntryId, in
                                             LedgerOffloaderStats offloaderStats, String ledgerName) {
         this(ledger, startEntryId, blockSize);
         this.offloaderStats = offloaderStats;
-        this.topicName = ledgerName;
+        this.managedLedgerName = ledgerName;
+        this.topicName = TopicName.fromPersistenceNamingEncoding(ledgerName);
     }
 
     private ByteBuf readEntries(int len) throws IOException {
@@ -183,7 +186,7 @@ private List readNextEntriesFromLedger(long start, long maxNumberEntrie
                 log.debug("read ledger entries. start: {}, end: {} cost {}", start, end,
                         TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - startTime));
             }
-            if (offloaderStats != null && topicName != null) {
+            if (offloaderStats != null && managedLedgerName != null) {
                 offloaderStats.recordReadLedgerLatency(topicName, System.nanoTime() - startTime,
                         TimeUnit.NANOSECONDS);
             }
diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java
index ef0bea29e35c3..ac87a8e424038 100644
--- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java
+++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java
@@ -49,6 +49,7 @@
 import org.apache.bookkeeper.mledger.OffloadedLedgerMetadata;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider;
 import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.options.CopyOptions;
 import org.mockito.Mockito;
@@ -172,10 +173,10 @@ public void testOffloadAndReadMetrics() throws Exception {
         LedgerOffloader offloader = getOffloader();
 
         UUID uuid = UUID.randomUUID();
-
-        String topic = "test";
+        String managedLegerName = "public/default/persistent/testOffload";
+        String topic = TopicName.fromPersistenceNamingEncoding(managedLegerName);
         Map extraMap = new HashMap<>();
-        extraMap.put("ManagedLedgerName", topic);
+        extraMap.put("ManagedLedgerName", managedLegerName);
         offloader.offload(toWrite, uuid, extraMap).get();
 
         LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats;
@@ -187,7 +188,7 @@ public void testOffloadAndReadMetrics() throws Exception {
 
         Map map = new HashMap<>();
         map.putAll(offloader.getOffloadDriverMetadata());
-        map.put("ManagedLedgerName", topic);
+        map.put("ManagedLedgerName", managedLegerName);
         ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get();
         LedgerEntries toTestEntries = toTest.read(0, toTest.getLastAddConfirmed());
         Iterator toTestIter = toTestEntries.iterator();

From 9f93af32d7844003d03cf754ec8cfc50b7177220 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 31 May 2023 16:11:30 +0800
Subject: [PATCH 351/404] [improve][broker] Do not expose
 bucketDelayedIndexStats (#20383)

---
 .../common/policies/data/stats/SubscriptionStatsImpl.java       | 2 ++
 .../pulsar/common/policies/data/stats/TopicStatsImpl.java       | 1 +
 2 files changed, 3 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index 2bfa54da6a002..cfc6cab9e1110 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.common.policies.data.stats;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -134,6 +135,7 @@ public class SubscriptionStatsImpl implements SubscriptionStats {
     /** The size of DelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
+    @JsonIgnore
     public Map bucketDelayedIndexStats;
 
     /** SubscriptionProperties (key/value strings) associated with this subscribe. */
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index f0141bd58d38f..7a48df89b8bcb 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -140,6 +140,7 @@ public class TopicStatsImpl implements TopicStats {
     public long delayedMessageIndexSizeInBytes;
 
     /** Map of bucket delayed index statistics. */
+    @JsonIgnore
     public Map bucketDelayedIndexStats;
 
     /** The compaction stats. */

From b3360745dc0db8c8a5ec60fb28cdb9e2718b29c6 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 31 May 2023 16:23:24 +0300
Subject: [PATCH 352/404] [improve][broker] Remove uneffective solution for
 reducing GC pressure (#20428)

---
 .../pulsar/broker/service/Consumer.java       |  6 +-
 .../pulsar/broker/service/Producer.java       |  6 +-
 .../pulsar/broker/service/ServerCnx.java      | 14 ++++
 .../pulsar/broker/service/TransportCnx.java   |  2 +
 .../data/stats/ConsumerStatsImpl.java         | 57 +++------------
 .../data/stats/PublisherStatsImpl.java        | 73 +++----------------
 6 files changed, 39 insertions(+), 119 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index a3f9da41e6b35..275d685280865 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -193,11 +193,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo
         this.metadata = metadata != null ? metadata : Collections.emptyMap();
 
         stats = new ConsumerStatsImpl();
-        if (cnx.hasHAProxyMessage()) {
-            stats.setAddress(cnx.getHAProxyMessage().sourceAddress() + ":" + cnx.getHAProxyMessage().sourcePort());
-        } else {
-            stats.setAddress(cnx.clientAddress().toString());
-        }
+        stats.setAddress(cnx.clientSourceAddressAndPort());
         stats.consumerName = consumerName;
         stats.setConnectedSince(DateFormatter.now());
         stats.setClientVersion(cnx.getClientVersion());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
index 53b79f06e8e24..f7d2bb2dd2797 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
@@ -127,11 +127,7 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN
         this.metadata = metadata != null ? metadata : Collections.emptyMap();
 
         this.stats = isNonPersistentTopic ? new NonPersistentPublisherStatsImpl() : new PublisherStatsImpl();
-        if (cnx.hasHAProxyMessage()) {
-            stats.setAddress(cnx.getHAProxyMessage().sourceAddress() + ":" + cnx.getHAProxyMessage().sourcePort());
-        } else {
-            stats.setAddress(cnx.clientAddress().toString());
-        }
+        stats.setAddress(cnx.clientSourceAddressAndPort());
         stats.setConnectedSince(DateFormatter.now());
         stats.setClientVersion(cnx.getClientVersion());
         stats.setProducerName(producerName);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index d3c3971dfaced..98c0e5b497998 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -215,6 +215,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
     private final String replicatorPrefix;
     private String clientVersion = null;
     private String proxyVersion = null;
+    private String clientSourceAddressAndPort;
     private int nonPersistentPendingMessages = 0;
     private final int maxNonPersistentPendingMessages;
     private String originalPrincipal = null;
@@ -3375,6 +3376,19 @@ public String clientSourceAddress() {
         }
     }
 
+    @Override
+    public String clientSourceAddressAndPort() {
+        if (clientSourceAddressAndPort == null) {
+            if (hasHAProxyMessage()) {
+                clientSourceAddressAndPort =
+                        getHAProxyMessage().sourceAddress() + ":" + getHAProxyMessage().sourcePort();
+            } else {
+                clientSourceAddressAndPort = clientAddress().toString();
+            }
+        }
+        return clientSourceAddressAndPort;
+    }
+
     CompletableFuture connectionCheckInProgress;
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
index d267160652ae4..94f934fec681e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
@@ -31,6 +31,8 @@ public interface TransportCnx {
 
     SocketAddress clientAddress();
 
+    String clientSourceAddressAndPort();
+
     BrokerService getBrokerService();
 
     PulsarCommandSender getCommandSender();
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java
index ddae2e7135695..548abdc9ada33 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pulsar.common.policies.data.stats;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -79,22 +78,11 @@ public class ConsumerStatsImpl implements ConsumerStats {
     public String readPositionWhenJoining;
 
     /** Address of this consumer. */
-    @JsonIgnore
-    private int addressOffset = -1;
-    @JsonIgnore
-    private int addressLength;
-
+    private String address;
     /** Timestamp of connection. */
-    @JsonIgnore
-    private int connectedSinceOffset = -1;
-    @JsonIgnore
-    private int connectedSinceLength;
-
+    private String connectedSince;
     /** Client library version. */
-    @JsonIgnore
-    private int clientVersionOffset = -1;
-    @JsonIgnore
-    private int clientVersionLength;
+    private String clientVersion;
 
     // ignore this json field to skip from stats in future release. replaced with readable #getLastAckedTime().
     @Deprecated
@@ -111,13 +99,6 @@ public class ConsumerStatsImpl implements ConsumerStats {
     /** Metadata (key/value strings) associated with this consumer. */
     public Map metadata;
 
-    /**
-     * In order to prevent multiple string object allocation under stats: create a string-buffer
-     * that stores data for all string place-holders.
-     */
-    @JsonIgnore
-    private StringBuilder stringBuffer = new StringBuilder();
-
     public ConsumerStatsImpl add(ConsumerStatsImpl stats) {
         Objects.requireNonNull(stats);
         this.msgRateOut += stats.msgRateOut;
@@ -134,47 +115,27 @@ public ConsumerStatsImpl add(ConsumerStatsImpl stats) {
     }
 
     public String getAddress() {
-        return addressOffset == -1 ? null : stringBuffer.substring(addressOffset, addressOffset + addressLength);
+        return address;
     }
 
     public void setAddress(String address) {
-        if (address == null) {
-            this.addressOffset = -1;
-            return;
-        }
-        this.addressOffset = this.stringBuffer.length();
-        this.addressLength = address.length();
-        this.stringBuffer.append(address);
+        this.address = address;
     }
 
     public String getConnectedSince() {
-        return connectedSinceOffset == -1 ? null
-                : stringBuffer.substring(connectedSinceOffset, connectedSinceOffset + connectedSinceLength);
+        return connectedSince;
     }
 
     public void setConnectedSince(String connectedSince) {
-        if (connectedSince == null) {
-            this.connectedSinceOffset = -1;
-            return;
-        }
-        this.connectedSinceOffset = this.stringBuffer.length();
-        this.connectedSinceLength = connectedSince.length();
-        this.stringBuffer.append(connectedSince);
+        this.connectedSince = connectedSince;
     }
 
     public String getClientVersion() {
-        return clientVersionOffset == -1 ? null
-                : stringBuffer.substring(clientVersionOffset, clientVersionOffset + clientVersionLength);
+        return clientVersion;
     }
 
     public void setClientVersion(String clientVersion) {
-        if (clientVersion == null) {
-            this.clientVersionOffset = -1;
-            return;
-        }
-        this.clientVersionOffset = this.stringBuffer.length();
-        this.clientVersionLength = clientVersion.length();
-        this.stringBuffer.append(clientVersion);
+        this.clientVersion = clientVersion;
     }
 
     public String getReadPositionWhenJoining() {
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java
index 41407a37e7ca0..304361bb2daec 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java
@@ -53,35 +53,13 @@ public class PublisherStatsImpl implements PublisherStats {
     public boolean supportsPartialProducer;
 
     /** Producer name. */
-    @JsonIgnore
-    private int producerNameOffset = -1;
-    @JsonIgnore
-    private int producerNameLength;
-
+    private String producerName;
     /** Address of this publisher. */
-    @JsonIgnore
-    private int addressOffset = -1;
-    @JsonIgnore
-    private int addressLength;
-
+    private String address;
     /** Timestamp of connection. */
-    @JsonIgnore
-    private int connectedSinceOffset = -1;
-    @JsonIgnore
-    private int connectedSinceLength;
-
+    private String connectedSince;
     /** Client library version. */
-    @JsonIgnore
-    private int clientVersionOffset = -1;
-    @JsonIgnore
-    private int clientVersionLength;
-
-    /**
-     * In order to prevent multiple string objects under stats: create a string-buffer that stores data for all string
-     * place-holders.
-     */
-    @JsonIgnore
-    private StringBuilder stringBuffer = new StringBuilder();
+    private String clientVersion;
 
     /** Metadata (key/value strings) associated with this publisher. */
     public Map metadata;
@@ -99,61 +77,34 @@ public PublisherStatsImpl add(PublisherStatsImpl stats) {
     }
 
     public String getProducerName() {
-        return producerNameOffset == -1 ? null
-                : stringBuffer.substring(producerNameOffset, producerNameOffset + producerNameLength);
+        return producerName;
     }
 
     public void setProducerName(String producerName) {
-        if (producerName == null) {
-            this.producerNameOffset = -1;
-            return;
-        }
-        this.producerNameOffset = this.stringBuffer.length();
-        this.producerNameLength = producerName.length();
-        this.stringBuffer.append(producerName);
+        this.producerName = producerName;
     }
 
     public String getAddress() {
-        return addressOffset == -1 ? null : stringBuffer.substring(addressOffset, addressOffset + addressLength);
+        return address;
     }
 
     public void setAddress(String address) {
-        if (address == null) {
-            this.addressOffset = -1;
-            return;
-        }
-        this.addressOffset = this.stringBuffer.length();
-        this.addressLength = address.length();
-        this.stringBuffer.append(address);
+        this.address = address;
     }
 
     public String getConnectedSince() {
-        return connectedSinceOffset == -1 ? null
-                : stringBuffer.substring(connectedSinceOffset, connectedSinceOffset + connectedSinceLength);
+        return connectedSince;
     }
 
     public void setConnectedSince(String connectedSince) {
-        if (connectedSince == null) {
-            this.connectedSinceOffset = -1;
-            return;
-        }
-        this.connectedSinceOffset = this.stringBuffer.length();
-        this.connectedSinceLength = connectedSince.length();
-        this.stringBuffer.append(connectedSince);
+        this.connectedSince = connectedSince;
     }
 
     public String getClientVersion() {
-        return clientVersionOffset == -1 ? null
-                : stringBuffer.substring(clientVersionOffset, clientVersionOffset + clientVersionLength);
+        return clientVersion;
     }
 
     public void setClientVersion(String clientVersion) {
-        if (clientVersion == null) {
-            this.clientVersionOffset = -1;
-            return;
-        }
-        this.clientVersionOffset = this.stringBuffer.length();
-        this.clientVersionLength = clientVersion.length();
-        this.stringBuffer.append(clientVersion);
+        this.clientVersion = clientVersion;
     }
 }

From 242758d5770de46e506855ff881472cbc274cedb Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 31 May 2023 22:57:16 +0300
Subject: [PATCH 353/404] [fix][test] Fix flaky PersistentSubscriptionTest
 (#20434)

---
 .../PersistentSubscriptionTest.java           | 106 +++++++++---------
 .../NonStartableTestPulsarService.java        |  17 +++
 2 files changed, 72 insertions(+), 51 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
index 401f52daa6291..87408598889e7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
@@ -92,7 +92,12 @@ public class PersistentSubscriptionTest {
     public void setup() throws Exception {
         pulsarTestContext = PulsarTestContext.builderForNonStartableContext()
                 .spyByDefault()
-                .configCustomizer(config -> config.setTransactionCoordinatorEnabled(true))
+                .configCustomizer(config -> {
+                    config.setTransactionCoordinatorEnabled(true);
+                    config.setTransactionPendingAckStoreProviderClassName(
+                            CustomTransactionPendingAckStoreProvider.class.getName());
+                    config.setTransactionBufferProviderClassName(InMemTransactionBufferProvider.class.getName());
+                })
                 .useTestPulsarResources()
                 .build();
 
@@ -100,56 +105,6 @@ public void setup() throws Exception {
         doReturn(Optional.of(new Policies())).when(namespaceResources)
                 .getPoliciesIfCached(any());
 
-        doReturn(new InMemTransactionBufferProvider()).when(pulsarTestContext.getPulsarService())
-                .getTransactionBufferProvider();
-        doReturn(new TransactionPendingAckStoreProvider() {
-            @Override
-            public CompletableFuture newPendingAckStore(PersistentSubscription subscription) {
-                return CompletableFuture.completedFuture(new PendingAckStore() {
-                    @Override
-                    public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) {
-                        try {
-                            Field field = PendingAckHandleState.class.getDeclaredField("state");
-                            field.setAccessible(true);
-                            field.set(pendingAckHandle, PendingAckHandleState.State.Ready);
-                        } catch (NoSuchFieldException | IllegalAccessException e) {
-                            fail();
-                        }
-                    }
-
-                    @Override
-                    public CompletableFuture closeAsync() {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-
-                    @Override
-                    public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) {
-                        return CompletableFuture.completedFuture(null);
-                    }
-                });
-            }
-
-            @Override
-            public CompletableFuture checkInitializedBefore(PersistentSubscription subscription) {
-                return CompletableFuture.completedFuture(true);
-            }
-        }).when(pulsarTestContext.getPulsarService()).getTransactionPendingAckStoreProvider();
-
         ledgerMock = mock(ManagedLedgerImpl.class);
         cursorMock = mock(ManagedCursorImpl.class);
         managedLedgerConfigMock = mock(ManagedLedgerConfig.class);
@@ -279,4 +234,53 @@ public void testAcknowledgeUpdateCursorLastActive() throws Exception {
         // `acknowledgeMessage` should update cursor last active
         assertTrue(persistentSubscription.cursor.getLastActive() > beforeAcknowledgeTimestamp);
     }
+
+    public static class CustomTransactionPendingAckStoreProvider implements TransactionPendingAckStoreProvider {
+        @Override
+        public CompletableFuture newPendingAckStore(PersistentSubscription subscription) {
+            return CompletableFuture.completedFuture(new PendingAckStore() {
+                @Override
+                public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) {
+                    try {
+                        Field field = PendingAckHandleState.class.getDeclaredField("state");
+                        field.setAccessible(true);
+                        field.set(pendingAckHandle, PendingAckHandleState.State.Ready);
+                    } catch (NoSuchFieldException | IllegalAccessException e) {
+                        fail();
+                    }
+                }
+
+                @Override
+                public CompletableFuture closeAsync() {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendIndividualAck(TxnID txnID,
+                                                                   List> positions) {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) {
+                    return CompletableFuture.completedFuture(null);
+                }
+
+                @Override
+                public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) {
+                    return CompletableFuture.completedFuture(null);
+                }
+            });
+        }
+
+        @Override
+        public CompletableFuture checkInitializedBefore(PersistentSubscription subscription) {
+            return CompletableFuture.completedFuture(true);
+        }
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java
index 4b7762a2acfdf..13c4d7d72af2c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java
@@ -21,6 +21,7 @@
 import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs;
 import static org.mockito.Mockito.mock;
 import io.netty.channel.EventLoopGroup;
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -39,6 +40,8 @@
 import org.apache.pulsar.broker.service.schema.DefaultSchemaRegistryService;
 import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
 import org.apache.pulsar.broker.storage.ManagedLedgerStorage;
+import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
@@ -89,6 +92,20 @@ public NonStartableTestPulsarService(SpyConfig spyConfig, ServiceConfiguration c
         } catch (PulsarServerException e) {
             throw new RuntimeException(e);
         }
+        if (config.isTransactionCoordinatorEnabled()) {
+            try {
+                setTransactionBufferProvider(TransactionBufferProvider
+                        .newProvider(config.getTransactionBufferProviderClassName()));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            try {
+                setTransactionPendingAckStoreProvider(TransactionPendingAckStoreProvider
+                        .newProvider(config.getTransactionPendingAckStoreProviderClassName()));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
     }
 
     @Override

From c35b820bb323c8e52bd9cd8ccd29565c23764117 Mon Sep 17 00:00:00 2001
From: crossoverJie 
Date: Thu, 1 Jun 2023 11:48:11 +0800
Subject: [PATCH 354/404] [fix][broker] Fix skip message API when hole messages
 exists (#20326)

---
 .../mledger/impl/ManagedCursorImpl.java       |  1 -
 .../pulsar/broker/admin/AdminApiTest.java     | 49 +++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 663081c932052..1ce0403a54762 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -1779,7 +1779,6 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) {
                 } finally {
                     if (r.lowerEndpoint() instanceof PositionImplRecyclable) {
                         ((PositionImplRecyclable) r.lowerEndpoint()).recycle();
-                        ((PositionImplRecyclable) r.upperEndpoint()).recycle();
                     }
                 }
             }, recyclePositionRangeConverter);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index 855343e18a24b..8561ea68c16e4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -55,11 +55,13 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.ws.rs.client.InvocationCallback;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response.Status;
 import lombok.Builder;
 import lombok.Cleanup;
+import lombok.SneakyThrows;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
@@ -90,6 +92,7 @@
 import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
@@ -993,6 +996,52 @@ public void persistentTopics(String topicName) throws Exception {
         assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>());
     }
 
+    @Test(dataProvider = "topicName")
+    public void testSkipHoleMessages(String topicName) throws Exception {
+        final String subName = topicName;
+        assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>());
+
+        final String persistentTopicName = "persistent://prop-xyz/ns1/" + topicName;
+        // Force to create a topic
+        publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 0);
+        assertEquals(admin.topics().getList("prop-xyz/ns1"),
+                List.of("persistent://prop-xyz/ns1/" + topicName));
+
+        // create consumer and subscription
+        @Cleanup
+        PulsarClient client = PulsarClient.builder()
+                .serviceUrl(pulsar.getWebServiceAddress())
+                .statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        AtomicInteger total = new AtomicInteger();
+        Consumer consumer = client.newConsumer().topic(persistentTopicName)
+                .messageListener(new MessageListener() {
+                    @SneakyThrows
+                    @Override
+                    public void received(Consumer consumer, Message msg) {
+                        if (total.get() %2 !=0){
+                            // artificially created 50 hollow messages
+                            consumer.acknowledge(msg);
+                        }
+                        total.incrementAndGet();
+                    }
+                })
+                .subscriptionName(subName)
+                .subscriptionType(SubscriptionType.Exclusive).subscribe();
+
+        assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName));
+
+        publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 100);
+        TimeUnit.SECONDS.sleep(2);
+        TopicStats topicStats = admin.topics().getStats(persistentTopicName);
+        long msgBacklog = topicStats.getSubscriptions().get(subName).getMsgBacklog();
+        log.info("back={}",msgBacklog);
+        int skipNumber = 20;
+        admin.topics().skipMessages(persistentTopicName, subName, skipNumber);
+        topicStats = admin.topics().getStats(persistentTopicName);
+        assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), msgBacklog - skipNumber);
+    }
+
     @Test(dataProvider = "topicNamesForAllTypes")
     public void partitionedTopics(String topicType, String topicName) throws Exception {
         final String namespace = "prop-xyz/ns1";

From 5e6e6cebcdbeec32ed49729f658f2d5cd0d98347 Mon Sep 17 00:00:00 2001
From: hleecs 
Date: Thu, 1 Jun 2023 12:36:46 +0800
Subject: [PATCH 355/404] [feat][broker]PIP-255 Part-1: Add listener interface
 for namespace service (#20406)

---
 .../NamespaceBundleSplitListener.java         | 29 ++++++++++++++++
 .../broker/namespace/NamespaceService.java    | 27 +++++++++++++++
 .../namespace/NamespaceCreateBundlesTest.java | 33 +++++++++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java
new file mode 100644
index 0000000000000..a3312f5689e38
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.namespace;
+
+import java.util.function.Predicate;
+import org.apache.pulsar.common.naming.NamespaceBundle;
+
+/**
+ * Listener for NamespaceBundle split.
+ */
+public interface NamespaceBundleSplitListener extends Predicate {
+    void onSplit(NamespaceBundle bundle);
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index e2d4ef5153769..cf969460c3345 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -139,6 +139,10 @@ public class NamespaceService implements AutoCloseable {
     private final ConcurrentOpenHashMap namespaceClients;
 
     private final List bundleOwnershipListeners;
+
+    private final List bundleSplitListeners;
+
+
     private final RedirectManager redirectManager;
 
 
@@ -167,6 +171,7 @@ public NamespaceService(PulsarService pulsar) {
         this.namespaceClients =
                 ConcurrentOpenHashMap.newBuilder().build();
         this.bundleOwnershipListeners = new CopyOnWriteArrayList<>();
+        this.bundleSplitListeners = new CopyOnWriteArrayList<>();
         this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class);
         this.redirectManager = new RedirectManager(pulsar);
     }
@@ -975,6 +980,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle,
                                 // affect the split operation which is already safely completed
                                 r.forEach(this::unloadNamespaceBundle);
                             }
+                            onNamespaceBundleSplit(bundle);
                         })
                         .exceptionally(e -> {
                             String msg1 = format(
@@ -1230,6 +1236,18 @@ protected void onNamespaceBundleUnload(NamespaceBundle bundle) {
         }
     }
 
+    protected void onNamespaceBundleSplit(NamespaceBundle bundle) {
+        for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) {
+            try {
+                if (bundleSplitListener.test(bundle)) {
+                    bundleSplitListener.onSplit(bundle);
+                }
+            } catch (Throwable t) {
+                LOG.error("Call bundle {} split listener {} error", bundle, bundleSplitListener, t);
+            }
+        }
+    }
+
     public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener... listeners) {
         Objects.requireNonNull(listeners);
         for (NamespaceBundleOwnershipListener listener : listeners) {
@@ -1240,6 +1258,15 @@ public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener
         getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners));
     }
 
+    public void addNamespaceBundleSplitListener(NamespaceBundleSplitListener... listeners) {
+        Objects.requireNonNull(listeners);
+        for (NamespaceBundleSplitListener listener : listeners) {
+            if (listener != null) {
+                bundleSplitListeners.add(listener);
+            }
+        }
+    }
+
     private void notifyNamespaceBundleOwnershipListener(NamespaceBundle bundle,
                     NamespaceBundleOwnershipListener... listeners) {
         if (listeners != null) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java
index 43d37466918ce..73cfaf1b0d96b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java
@@ -20,15 +20,21 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import lombok.Cleanup;
 
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.pulsar.broker.service.BrokerTestBase;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerBuilder;
+import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.policies.data.BookieAffinityGroupData;
 import org.apache.pulsar.common.policies.data.Policies;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -81,4 +87,31 @@ public void testSplitBundleUpdatesLocalPoliciesWithoutOverwriting() throws Excep
         assertNotNull(admin.namespaces().getBookieAffinityGroup(namespaceName));
         producer.close();
     }
+
+    @Test
+    public void testBundleSplitListener() throws Exception {
+        String namespaceName = "prop/" + UUID.randomUUID().toString();
+        String topicName = "persistent://" + namespaceName + "/my-topic5";
+        admin.namespaces().createNamespace(namespaceName);
+        @Cleanup
+        Producer producer = pulsarClient.newProducer().topic(topicName).sendTimeout(1,
+            TimeUnit.SECONDS).create();
+        producer.send(new byte[1]);
+        String bundleRange = admin.lookups().getBundleRange(topicName);
+        AtomicBoolean isTriggered = new AtomicBoolean(false);
+        pulsar.getNamespaceService().addNamespaceBundleSplitListener(new NamespaceBundleSplitListener() {
+            @Override
+            public void onSplit(NamespaceBundle bundle) {
+                assertEquals(bundleRange, bundle.getBundleRange());
+                isTriggered.set(true);
+            }
+
+            @Override
+            public boolean test(NamespaceBundle namespaceBundle) {
+                return true;
+            }
+        });
+        admin.namespaces().splitNamespaceBundle(namespaceName, bundleRange, false, null);
+        Awaitility.await().untilAsserted(() -> assertTrue(isTriggered.get()));
+    }
 }

From f86e36fcf1b68afd7ced44cc09b9028f0bb5846c Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Fri, 2 Jun 2023 00:37:18 +0800
Subject: [PATCH 356/404] [cleanup][ml] cleanup numManagedLedgerWorkerThreads
 (#20454)

---
 .../apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
index 8a4b4d4013f8f..386310b3ccbae 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
@@ -39,7 +39,6 @@ public class ManagedLedgerFactoryConfig {
      */
     private double cacheEvictionWatermark = 0.90;
 
-    private int numManagedLedgerWorkerThreads = Runtime.getRuntime().availableProcessors();
     private int numManagedLedgerSchedulerThreads = Runtime.getRuntime().availableProcessors();
 
     /**

From e05b890b804cee781e87db750f2afc3a382fa1d2 Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Thu, 1 Jun 2023 13:42:04 -0400
Subject: [PATCH 357/404] [fix][fn]Reset idle timer correctly (#20450)

Co-authored-by: Andy Walker 

Fix apache/pulsar#20449



Master Issue: #20449

### Verifying this change

- [ ] Make sure that the change passes the CI checks.

This change is a trivial rework / code cleanup without any test coverage.

### Does this pull request potentially affect one of the following parts:



*If the box was checked, please highlight the changes*

- [ ] Dependencies (add or upgrade a dependency)
- [ ] The public API
- [ ] The schema
- [ ] The default values of configurations
- [ ] The threading model
- [ ] The binary protocol
- [ ] The REST endpoints
- [ ] The admin CLI options
- [ ] The metrics
- [ ] Anything that affects deployment

### Documentation



- [ ] `doc` 
- [ ] `doc-required` 
- [X] `doc-not-needed` 
- [ ] `doc-complete` 

### Matching PR in forked repository

PR in forked repository: https://github.com/flowchartsman/pulsar/pull/5
---
 pulsar-function-go/pf/instance.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 6f73f3e631212..2c2deb19b2813 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -149,7 +149,6 @@ func (gi *goInstance) startFunction(function function) error {
 	defer metricsServicer.close()
 CLOSE:
 	for {
-		idleTimer.Reset(idleDuration)
 		select {
 		case cm := <-channel:
 			msgInput := cm.Message
@@ -181,6 +180,11 @@ CLOSE:
 			close(channel)
 			break CLOSE
 		}
+		// reset the idle timer and drain if appropriate before the next loop
+		if !idleTimer.Stop() {
+			<-idleTimer.C
+		}
+		idleTimer.Reset(idleDuration)
 	}
 
 	gi.closeLogTopic()

From 57f9467a8dbcd546ee9127d8dfbd000b46333f23 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 1 Jun 2023 22:19:49 +0300
Subject: [PATCH 358/404] [fix][sec] Upgrade Guava to 32.0.0 to address
 CVE-2023-2976 (#20459)

---
 buildtools/pom.xml                               | 2 +-
 distribution/server/src/assemble/LICENSE.bin.txt | 4 ++--
 distribution/shell/src/assemble/LICENSE.bin.txt  | 4 ++--
 pom.xml                                          | 2 +-
 pulsar-sql/presto-distribution/LICENSE           | 4 ++--
 pulsar-sql/presto-distribution/pom.xml           | 2 +-
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index 5a391777f2567..329eb9de6b552 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -49,7 +49,7 @@
     3.1.2
     4.1.93.Final
     4.2.3
-    31.0.1-jre
+    32.0.0-jre
     1.10.12
     2.0
     3.12.4
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 487e4e96b6a66..320f55703c99b 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -265,7 +265,7 @@ The Apache Software License, Version 2.0
     - com.google.code.gson-gson-2.8.9.jar
     - io.gsonfire-gson-fire-1.8.5.jar
  * Guava
-    - com.google.guava-guava-31.0.1-jre.jar
+    - com.google.guava-guava-32.0.0-jre.jar
     - com.google.guava-failureaccess-1.0.1.jar
     - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
  * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar
@@ -518,7 +518,7 @@ MIT License
     - org.slf4j-slf4j-api-1.7.32.jar
     - org.slf4j-jcl-over-slf4j-1.7.32.jar
  * The Checker Framework
-    - org.checkerframework-checker-qual-3.12.0.jar
+    - org.checkerframework-checker-qual-3.33.0.jar
  * oshi
     - com.github.oshi-oshi-core-java11-6.4.0.jar
  * Auth0, Inc.
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index c04ac2b7d0363..21ae10d0d5373 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -326,7 +326,7 @@ The Apache Software License, Version 2.0
  * Gson
     - gson-2.8.9.jar
  * Guava
-    - guava-31.0.1-jre.jar
+    - guava-32.0.0-jre.jar
     - failureaccess-1.0.1.jar
     - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
  * J2ObjC Annotations -- j2objc-annotations-1.3.jar
@@ -422,7 +422,7 @@ MIT License
  * SLF4J -- ../licenses/LICENSE-SLF4J.txt
     - slf4j-api-1.7.32.jar
  * The Checker Framework
-    - checker-qual-3.12.0.jar
+    - checker-qual-3.33.0.jar
 
 Protocol Buffers License
  * Protocol Buffers
diff --git a/pom.xml b/pom.xml
index c1a126b1161c0..31c1a1090a4e9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -202,7 +202,7 @@ flexible messaging model and an intuitive client API.
     2.10.2
     3.3.5
     2.4.16
-    31.0.1-jre
+    32.0.0-jre
     1.0
     0.16.1
     6.2.8
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 2a13985ac4ed5..67fcccc45bd7c 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -221,7 +221,7 @@ The Apache Software License, Version 2.0
     - jackson-module-jaxb-annotations-2.14.2.jar
     - jackson-module-jsonSchema-2.14.2.jar
  * Guava
-    - guava-31.0.1-jre.jar
+    - guava-32.0.0-jre.jar
     - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
     - failureaccess-1.0.1.jar
  * Google Guice
@@ -521,7 +521,7 @@ MIT License
  * JCL 1.2 Implemented Over SLF4J
    - jcl-over-slf4j-1.7.32.jar
  * Checker Qual
-   - checker-qual-3.12.0.jar
+   - checker-qual-3.33.0.jar
  * ScribeJava
    - scribejava-apis-6.9.0.jar
    - scribejava-core-6.9.0.jar
diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml
index e33a5733bbefb..8335aa3603f63 100644
--- a/pulsar-sql/presto-distribution/pom.xml
+++ b/pulsar-sql/presto-distribution/pom.xml
@@ -37,7 +37,7 @@
     2.6
     0.0.12
     3.0.5
-    31.0.1-jre
+    32.0.0-jre
     2.12.1
     2.5.1
     4.0.1

From a5f2d25e0f435fa8758904a0f71ceefceb8b5deb Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 1 Jun 2023 23:07:11 +0300
Subject: [PATCH 359/404] [fix][test] Remove dependency on httpbin.org service
 in FunctionCommonTest (#20464)

---
 pulsar-functions/utils/pom.xml                   |  9 ++++++++-
 .../functions/utils/FunctionCommonTest.java      | 16 +++++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml
index c6d0ceec3b395..327d14731641c 100644
--- a/pulsar-functions/utils/pom.xml
+++ b/pulsar-functions/utils/pom.xml
@@ -99,6 +99,13 @@
       ${project.version}
     
 
+    
+      com.github.tomakehurst
+      wiremock-jre8
+      ${wiremock.version}
+      test
+    
+
   
   
     
@@ -119,7 +126,7 @@
           
         
       
-      
+
       
         org.apache.maven.plugins
         maven-checkstyle-plugin
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
index 113824fc7c1a1..131f153b08d68 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
@@ -18,12 +18,19 @@
  */
 package org.apache.pulsar.functions.utils;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.configureFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
+import com.github.tomakehurst.wiremock.WireMockServer;
 import java.io.File;
 import java.util.Collection;
+import lombok.Cleanup;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.functions.api.Context;
@@ -87,7 +94,14 @@ public void testDownloadFile() throws Exception {
 
     @Test
     public void testDownloadFileWithBasicAuth() throws Exception {
-        final String jarHttpUrl = "https://foo:bar@httpbin.org/basic-auth/foo/bar";
+        @Cleanup("stop")
+        WireMockServer server = new WireMockServer(0);
+        server.start();
+        configureFor(server.port());
+        stubFor(get(urlPathEqualTo("/"))
+                .withBasicAuth("foo", "bar")
+                .willReturn(aResponse().withBody("Hello world!").withStatus(200)));
+        final String jarHttpUrl = "http://foo:bar@localhost:" + server.port() + "/";
         final File file = Files.newTemporaryFile();
         file.deleteOnExit();
         assertThat(file.length()).isZero();

From 2d7c3b0e47a40f3ca265602d057859b77077dcc0 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Thu, 1 Jun 2023 23:07:32 +0300
Subject: [PATCH 360/404] [fix][sec] Upgrade Jetty to address CVEs (#20461)

---
 .../server/src/assemble/LICENSE.bin.txt       | 38 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        | 16 ++++----
 pom.xml                                       |  2 +-
 pulsar-sql/presto-distribution/LICENSE        | 32 ++++++++--------
 4 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 320f55703c99b..d7033ac85646e 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -383,25 +383,25 @@ The Apache Software License, Version 2.0
     - org.asynchttpclient-async-http-client-2.12.1.jar
     - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - org.eclipse.jetty-jetty-client-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-continuation-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-http-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-io-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-proxy-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-security-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-server-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-servlet-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-servlets-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-util-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-util-ajax-9.4.48.v20220622.jar
-    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.48.v20220622.jar
-    - org.eclipse.jetty.websocket-websocket-api-9.4.48.v20220622.jar
-    - org.eclipse.jetty.websocket-websocket-client-9.4.48.v20220622.jar
-    - org.eclipse.jetty.websocket-websocket-common-9.4.48.v20220622.jar
-    - org.eclipse.jetty.websocket-websocket-server-9.4.48.v20220622.jar
-    - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar
-    - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar
+    - org.eclipse.jetty-jetty-client-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-continuation-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-http-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-io-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-proxy-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-security-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-server-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-servlet-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-servlets-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-util-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-util-ajax-9.4.51.v20230217.jar
+    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.51.v20230217.jar
+    - org.eclipse.jetty.websocket-websocket-api-9.4.51.v20230217.jar
+    - org.eclipse.jetty.websocket-websocket-client-9.4.51.v20230217.jar
+    - org.eclipse.jetty.websocket-websocket-common-9.4.51.v20230217.jar
+    - org.eclipse.jetty.websocket-websocket-server-9.4.51.v20230217.jar
+    - org.eclipse.jetty.websocket-websocket-servlet-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-alpn-server-9.4.51.v20230217.jar
  * SnakeYaml -- org.yaml-snakeyaml-2.0.jar
  * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 21ae10d0d5373..514bb41a9499c 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -399,14 +399,14 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - jetty-client-9.4.48.v20220622.jar
-    - jetty-http-9.4.48.v20220622.jar
-    - jetty-io-9.4.48.v20220622.jar
-    - jetty-util-9.4.48.v20220622.jar
-    - javax-websocket-client-impl-9.4.48.v20220622.jar
-    - websocket-api-9.4.48.v20220622.jar
-    - websocket-client-9.4.48.v20220622.jar
-    - websocket-common-9.4.48.v20220622.jar
+    - jetty-client-9.4.51.v20230217.jar
+    - jetty-http-9.4.51.v20230217.jar
+    - jetty-io-9.4.51.v20230217.jar
+    - jetty-util-9.4.51.v20230217.jar
+    - javax-websocket-client-impl-9.4.51.v20230217.jar
+    - websocket-api-9.4.51.v20230217.jar
+    - websocket-client-9.4.51.v20230217.jar
+    - websocket-common-9.4.51.v20230217.jar
  * SnakeYaml -- snakeyaml-2.0.jar
  * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar
  * Javassist -- javassist-3.25.0-GA.jar
diff --git a/pom.xml b/pom.xml
index 31c1a1090a4e9..699fc1631c846 100644
--- a/pom.xml
+++ b/pom.xml
@@ -142,7 +142,7 @@ flexible messaging model and an intuitive client API.
     5.1.0
     4.1.93.Final
     0.0.21.Final
-    9.4.48.v20220622
+    9.4.51.v20230217
     2.5.2
     2.34
     1.10.50
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 67fcccc45bd7c..9dddc0d77bcf1 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -283,22 +283,22 @@ The Apache Software License, Version 2.0
     - joda-time-2.10.10.jar
     - failsafe-2.4.4.jar
   * Jetty
-    - http2-client-9.4.48.v20220622.jar
-    - http2-common-9.4.48.v20220622.jar
-    - http2-hpack-9.4.48.v20220622.jar
-    - http2-http-client-transport-9.4.48.v20220622.jar
-    - jetty-alpn-client-9.4.48.v20220622.jar
-    - http2-server-9.4.48.v20220622.jar
-    - jetty-alpn-java-client-9.4.48.v20220622.jar
-    - jetty-client-9.4.48.v20220622.jar
-    - jetty-http-9.4.48.v20220622.jar
-    - jetty-io-9.4.48.v20220622.jar
-    - jetty-jmx-9.4.48.v20220622.jar
-    - jetty-security-9.4.48.v20220622.jar
-    - jetty-server-9.4.48.v20220622.jar
-    - jetty-servlet-9.4.48.v20220622.jar
-    - jetty-util-9.4.48.v20220622.jar
-    - jetty-util-ajax-9.4.48.v20220622.jar
+    - http2-client-9.4.51.v20230217.jar
+    - http2-common-9.4.51.v20230217.jar
+    - http2-hpack-9.4.51.v20230217.jar
+    - http2-http-client-transport-9.4.51.v20230217.jar
+    - jetty-alpn-client-9.4.51.v20230217.jar
+    - http2-server-9.4.51.v20230217.jar
+    - jetty-alpn-java-client-9.4.51.v20230217.jar
+    - jetty-client-9.4.51.v20230217.jar
+    - jetty-http-9.4.51.v20230217.jar
+    - jetty-io-9.4.51.v20230217.jar
+    - jetty-jmx-9.4.51.v20230217.jar
+    - jetty-security-9.4.51.v20230217.jar
+    - jetty-server-9.4.51.v20230217.jar
+    - jetty-servlet-9.4.51.v20230217.jar
+    - jetty-util-9.4.51.v20230217.jar
+    - jetty-util-ajax-9.4.51.v20230217.jar
   * Byte Buddy
     - byte-buddy-1.11.13.jar
   * Apache BVal

From 174c4866f926b1ae7f7935395fbea8c0aeb64e31 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Thu, 1 Jun 2023 21:37:32 -0500
Subject: [PATCH 361/404] [fix][test] Replace test call to Auth0 with call to
 WireMock (#20465)

---
 pulsar-broker/pom.xml                         |   7 ++
 ...uth2AuthenticatedProducerConsumerTest.java | 119 ++++++++++++++++--
 2 files changed, 115 insertions(+), 11 deletions(-)

diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index b327375613c59..e084d70c79ad6 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -151,6 +151,13 @@
       test
     
 
+    
+      com.github.tomakehurst
+      wiremock-jre8
+      ${wiremock.version}
+      test
+    
+
     
     
        io.dropwizard.metrics
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java
index 22834b2e0c9c1..cf85ddd913b8b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java
@@ -18,15 +18,34 @@
  */
 package org.apache.pulsar.client.api;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
 import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.common.FileSource;
+import com.github.tomakehurst.wiremock.extension.Parameters;
+import com.github.tomakehurst.wiremock.extension.ResponseTransformer;
+import com.github.tomakehurst.wiremock.http.Request;
+import com.github.tomakehurst.wiremock.http.Response;
 import com.google.common.collect.Sets;
+import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.impl.DefaultJwtBuilder;
+import io.jsonwebtoken.security.Keys;
 import java.net.URI;
 import java.net.URL;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.KeyPair;
+import java.security.PrivateKey;
 import java.time.Duration;
+import java.util.Base64;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.Properties;
 import java.util.Set;
@@ -55,22 +74,51 @@
 public class TokenOauth2AuthenticatedProducerConsumerTest extends ProducerConsumerBase {
     private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class);
 
-    // public key in oauth2 server to verify the client passed in token. get from https://jwt.io/
-    private final String TOKEN_TEST_PUBLIC_KEY = "data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRmWjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDHy5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB";
+    private WireMockServer server;
 
     private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients";
 
     // Credentials File, which contains "client_id" and "client_secret"
     private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json";
-    private final String ISSUER_URL = "https://dev-kt-aa9ne.us.auth0.com";
     private final String AUDIENCE = "https://dev-kt-aa9ne.us.auth0.com/api/v2/";
 
     @BeforeMethod(alwaysRun = true)
     @Override
     protected void setup() throws Exception {
+        // Create the token key pair
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+
+        // Start mocked OAuth2 server
+        server = new WireMockServer(wireMockConfig().port(0).extensions(new OAuth2Transformer(keyPair, 3000)));
+        server.start();
+
+        // Set up a correct openid-configuration that points to the next stub
+        server.stubFor(
+                get(urlEqualTo("/.well-known/openid-configuration"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "%s",
+                                          "token_endpoint": "%s/oauth/token"
+                                        }
+                                        """.replace("%s", server.baseUrl()))));
+
+        // Only respond when the client sends the expected request body
+        server.stubFor(
+                post(urlEqualTo("/oauth/token"))
+                        .withRequestBody(
+                                equalTo("audience=https%3A%2F%2Fdev-kt-aa9ne.us.auth0.com%2Fapi%2Fv2%2F&"
+                                        + "client_id=Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x&"
+                                        + "client_secret=rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N"
+                                        + "_poGAb&grant_type=client_credentials"))
+                        .willReturn(aResponse()
+                                .withTransformers("o-auth-token-transformer")
+                                .withStatus(200)));
+
         conf.setAuthenticationEnabled(true);
         conf.setAuthorizationEnabled(true);
-        conf.setAuthenticationRefreshCheckSeconds(5);
+        conf.setAuthenticationRefreshCheckSeconds(1);
 
         Set superUserRoles = new HashSet<>();
         superUserRoles.add(ADMIN_ROLE);
@@ -83,7 +131,7 @@ protected void setup() throws Exception {
         conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName());
         conf.setBrokerClientAuthenticationParameters("{\n"
                 + "  \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n"
-                + "  \"issuerUrl\": \"" + ISSUER_URL + "\",\n"
+                + "  \"issuerUrl\": \"" + server.baseUrl() + "\",\n"
                 + "  \"audience\": \"" + AUDIENCE + "\",\n"
                 + "}\n");
 
@@ -91,7 +139,8 @@ protected void setup() throws Exception {
 
         // Set provider domain name
         Properties properties = new Properties();
-        properties.setProperty("tokenPublicKey", TOKEN_TEST_PUBLIC_KEY);
+        properties.setProperty("tokenPublicKey", "data:;base64,"
+                + Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded()));
 
         conf.setProperties(properties);
         super.init();
@@ -104,7 +153,7 @@ protected final void clientSetup() throws Exception {
 
         // AuthenticationOAuth2
         Authentication authentication = AuthenticationFactoryOAuth2.clientCredentials(
-                new URL(ISSUER_URL),
+                new URL(server.baseUrl()),
                 path.toUri().toURL(),  // key file path
                 AUDIENCE
         );
@@ -122,6 +171,7 @@ protected final void clientSetup() throws Exception {
     @Override
     protected void cleanup() throws Exception {
         super.internalCleanup();
+        server.stop();
     }
 
     @DataProvider(name = "batch")
@@ -210,12 +260,11 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception {
         String accessTokenOld = producerImpl.getClientCnx().getAuthenticationDataProvider().getCommandData();
         long lastDisconnectTime = producer.getLastDisconnectedTimestamp();
 
-        // the token expire duration is 10 seconds, so we need to wait for the authenticationData refreshed
+        // the token expire duration is 3 seconds, so we need to wait for the authenticationData refreshed
         Awaitility.await()
-            .atLeast(10, TimeUnit.SECONDS)
-            .atMost(20, TimeUnit.SECONDS)
+            .atMost(10, TimeUnit.SECONDS)
             .with()
-            .pollInterval(Duration.ofSeconds(1))
+            .pollInterval(Duration.ofMillis(250))
             .untilAsserted(() -> {
                 String accessTokenNew = producerImpl.getClientCnx().getAuthenticationDataProvider().getCommandData();
                 assertNotEquals(accessTokenNew, accessTokenOld);
@@ -243,4 +292,52 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception {
         consumer.acknowledgeCumulative(msg);
         consumer.close();
     }
+
+    class OAuth2Transformer extends ResponseTransformer {
+
+        private final PrivateKey privateKey;
+        private final long tokenTTL;
+
+        OAuth2Transformer(KeyPair key, long tokenTTLMillis) {
+            this.privateKey = key.getPrivate();
+            this.tokenTTL = tokenTTLMillis;
+        }
+
+        @Override
+        public Response transform(Request request, Response response, FileSource files, Parameters parameters) {
+            return Response.Builder.like(response).but().body("""
+                                                              {
+                                                                "access_token": "%s",
+                                                                "expires_in": %d,
+                                                                "token_type":"Bearer"
+                                                              }
+                                                              """.formatted(generateToken(),
+                    TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build();
+        }
+
+        @Override
+        public String getName() {
+            return "o-auth-token-transformer";
+        }
+
+        @Override
+        public boolean applyGlobally() {
+            return false;
+        }
+
+        private String generateToken() {
+            long now = System.currentTimeMillis();
+            DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+            defaultJwtBuilder.setHeaderParam("typ", "JWT");
+            defaultJwtBuilder.setHeaderParam("alg", "RS256");
+            defaultJwtBuilder.setIssuer(server.baseUrl());
+            defaultJwtBuilder.setSubject(ADMIN_ROLE);
+            defaultJwtBuilder.setAudience(AUDIENCE);
+            defaultJwtBuilder.setIssuedAt(new Date(now));
+            defaultJwtBuilder.setNotBefore(new Date(now));
+            defaultJwtBuilder.setExpiration(new Date(now + tokenTTL));
+            defaultJwtBuilder.signWith(privateKey);
+            return defaultJwtBuilder.compact();
+        }
+    }
 }

From 49174a90170d936c7d108894917a277cf292b97c Mon Sep 17 00:00:00 2001
From: Masahiro Sakamoto 
Date: Fri, 2 Jun 2023 14:11:11 +0900
Subject: [PATCH 362/404] [improve][broker] Log resource usage rate of brokers
 that need to be offloaded in ThresholdShedder (#19983)

---
 .../broker/loadbalance/impl/ThresholdShedder.java | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java
index 86df49f952674..882c72a71c904 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java
@@ -79,7 +79,8 @@ public synchronized Multimap findBundlesForUnloading(final LoadD
             final double currentUsage = brokerAvgResourceUsage.getOrDefault(broker, 0.0);
             if (currentUsage < avgUsage + threshold) {
                 if (log.isDebugEnabled()) {
-                    log.debug("[{}] broker is not overloaded, ignoring at this point", broker);
+                    log.debug("[{}] broker is not overloaded, ignoring at this point ({})", broker,
+                            localData.printResourceUsage());
                 }
                 return;
             }
@@ -92,17 +93,19 @@ public synchronized Multimap findBundlesForUnloading(final LoadD
             if (minimumThroughputToOffload < minThroughputThreshold) {
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] broker is planning to shed throughput {} MByte/s less than "
-                                    + "minimumThroughputThreshold {} MByte/s, skipping bundle unload.",
-                            broker, minimumThroughputToOffload / MB, minThroughputThreshold / MB);
+                                    + "minimumThroughputThreshold {} MByte/s, skipping bundle unload ({})",
+                            broker, minimumThroughputToOffload / MB, minThroughputThreshold / MB,
+                            localData.printResourceUsage());
                 }
                 return;
             }
 
             log.info(
-                    "Attempting to shed load on {}, which has max resource usage above avgUsage  and threshold {}%"
-                            + " > {}% + {}% -- Offloading at least {} MByte/s of traffic, left throughput {} MByte/s",
+                    "Attempting to shed load on {}, which has max resource usage above avgUsage and threshold {}%"
+                            + " > {}% + {}% -- Offloading at least {} MByte/s of traffic,"
+                                    + " left throughput {} MByte/s ({})",
                     broker, 100 * currentUsage, 100 * avgUsage, 100 * threshold, minimumThroughputToOffload / MB,
-                    (brokerCurrentThroughput - minimumThroughputToOffload) / MB);
+                    (brokerCurrentThroughput - minimumThroughputToOffload) / MB, localData.printResourceUsage());
 
             if (localData.getBundles().size() > 1) {
                 filterAndSelectBundle(loadData, recentlyUnloadedBundles, broker, localData, minimumThroughputToOffload);

From e220a5d04ae16d1b8dfd7e35cdddf43f3a43fe86 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Fri, 2 Jun 2023 14:16:44 +0800
Subject: [PATCH 363/404] [improve][broker] Support revoking permission for
 AuthorizationProvider (#20456)

---
 .../authorization/AuthorizationProvider.java  | 29 ++++++++++-
 .../authorization/AuthorizationService.java   | 29 +++++++++--
 .../PulsarAuthorizationProvider.java          | 50 +++++++++++++++++++
 .../AuthorizationProducerConsumerTest.java    | 48 ++++++++++++++++++
 4 files changed, 150 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java
index 67e096bee63e4..b54b2089e1eb1 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java
@@ -185,6 +185,18 @@ CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String
     CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role,
             String authDataJson);
 
+    /**
+     * Revoke authorization-action permission on a namespace to the given client.
+     * @param namespace
+     * @param role
+     * @return CompletableFuture
+     */
+    default CompletableFuture revokePermissionAsync(NamespaceName namespace, String role) {
+        return FutureUtil.failedFuture(new IllegalStateException(
+                String.format("revokePermissionAsync on namespace %s is not supported by the Authorization",
+                        namespace)));
+    }
+
     /**
      * Grant permission to roles that can access subscription-admin api.
      *
@@ -193,7 +205,7 @@ CompletableFuture grantPermissionAsync(NamespaceName namespace, Set
      */
     CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
                                                              Set roles, String authDataJson);
@@ -203,7 +215,7 @@ CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace
      * @param namespace
      * @param subscriptionName
      * @param role
-     * @return
+     * @return CompletableFuture
      */
     CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
             String role, String authDataJson);
@@ -226,6 +238,19 @@ CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespac
     CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role,
             String authDataJson);
 
+
+    /**
+     * Revoke authorization-action permission on a topic to the given client.
+     * @param topicName
+     * @param role
+     * @return CompletableFuture
+     */
+    default CompletableFuture revokePermissionAsync(TopicName topicName, String role) {
+        return FutureUtil.failedFuture(new IllegalStateException(
+                String.format("revokePermissionAsync on topicName %s is not supported by the Authorization",
+                        topicName)));
+    }
+
     /**
      * Check if a given role is allowed to execute a given operation on the tenant.
      *
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index 6f303e2117fe0..29abcc1eee414 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -123,6 +123,17 @@ public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set
         return provider.grantPermissionAsync(namespace, actions, role, authDataJson);
     }
 
+    /**
+     *
+     * Revoke authorization-action permission on a namespace to the given client.
+     *
+     * @param namespace
+     * @param role
+     */
+    public CompletableFuture revokePermissionAsync(NamespaceName namespace, String role) {
+        return provider.revokePermissionAsync(namespace, role);
+    }
+
     /**
      * Grant permission to roles that can access subscription-admin api.
      *
@@ -157,16 +168,26 @@ public CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName n
      * NOTE: used to complete with {@link IllegalArgumentException} when namespace not found or with
      * {@link IllegalStateException} when failed to grant permission.
      *
-     * @param topicname
+     * @param topicName
      * @param role
      * @param authDataJson
      *            additional authdata in json for targeted authorization provider
      * @completesWith null when the permissions are updated successfully.
      * @completesWith {@link MetadataStoreException} when the MetadataStore is not updated.
      */
-    public CompletableFuture grantPermissionAsync(TopicName topicname, Set actions, String role,
+    public CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role,
                                                         String authDataJson) {
-        return provider.grantPermissionAsync(topicname, actions, role, authDataJson);
+        return provider.grantPermissionAsync(topicName, actions, role, authDataJson);
+    }
+
+    /**
+     * Revoke authorization-action permission on a topic to the given client.
+     *
+     * @param topicName
+     * @param role
+     */
+    public CompletableFuture revokePermissionAsync(TopicName topicName, String role) {
+        return provider.revokePermissionAsync(topicName, role);
     }
 
     /**
@@ -418,7 +439,7 @@ private boolean isValidOriginalPrincipal(AuthenticationParameters authParams) {
     /**
      * Whether the authenticatedPrincipal and the originalPrincipal form a valid pair. This method assumes that
      * authenticatedPrincipal and originalPrincipal can be equal, as long as they are not a proxy role. This use
-     * case is relvant for the admin server because of the way the proxy handles authentication. The binary protocol
+     * case is relevant for the admin server because of the way the proxy handles authentication. The binary protocol
      * should not use this method.
      * @return true when roles are a valid combination and false when roles are an invalid combination
      */
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
index 3f6d38194713e..203f7fe427745 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
@@ -249,6 +249,33 @@ public CompletableFuture grantPermissionAsync(TopicName topicName, Set revokePermissionAsync(TopicName topicName, String role) {
+        return getPoliciesReadOnlyAsync().thenCompose(readonly -> {
+            if (readonly) {
+                if (log.isDebugEnabled()) {
+                    log.debug("Policies are read-only. Broker cannot do read-write operations");
+                }
+                throw new IllegalStateException("policies are in readonly mode");
+            }
+            return pulsarResources.getNamespaceResources()
+                    .setPoliciesAsync(topicName.getNamespaceObject(), policies -> {
+                        policies.auth_policies.getTopicAuthentication()
+                                .computeIfPresent(topicName.toString(), (k, v) -> {
+                                        v.remove(role);
+                                        return null;
+                                });
+                        return policies;
+                    }).whenComplete((__, ex) -> {
+                        if (ex != null) {
+                            log.error("Failed to revoke permissions for role {} on topic {}", role, topicName, ex);
+                        } else {
+                            log.info("Successfully revoke permissions for role {} on topic {}", role, topicName);
+                        }
+                    });
+        });
+    }
+
     @Override
     public CompletableFuture grantPermissionAsync(NamespaceName namespaceName, Set actions,
                                                         String role, String authDataJson) {
@@ -274,6 +301,29 @@ public CompletableFuture grantPermissionAsync(NamespaceName namespaceName,
         });
     }
 
+    @Override
+    public CompletableFuture revokePermissionAsync(NamespaceName namespaceName, String role) {
+        return getPoliciesReadOnlyAsync().thenCompose(readonly -> {
+            if (readonly) {
+                if (log.isDebugEnabled()) {
+                    log.debug("Policies are read-only. Broker cannot do read-write operations");
+                }
+                throw new IllegalStateException("policies are in readonly mode");
+            }
+            return pulsarResources.getNamespaceResources()
+                    .setPoliciesAsync(namespaceName, policies -> {
+                        policies.auth_policies.getNamespaceAuthentication().remove(role);
+                        return policies;
+                    }).whenComplete((__, ex) -> {
+                        if (ex != null) {
+                            log.error("Failed to revoke permissions for role {} namespace {}", role, namespaceName, ex);
+                        } else {
+                            log.info("Successfully revoke permissions for role {} namespace {}", role, namespaceName);
+                        }
+                    });
+        });
+    }
+
     @Override
     public CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
                                                                     Set roles, String authDataJson) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
index 0ce3b7df07d1f..01fa64b1bc62e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
@@ -667,6 +667,54 @@ public void testGrantPermission() throws Exception {
         log.info("-- Exiting {} test --", methodName);
     }
 
+    @Test
+    public void testRevokePermission() throws Exception {
+        log.info("-- Starting {} test --", methodName);
+        cleanup();
+        conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName());
+        setup();
+
+        Authentication adminAuthentication = new ClientAuthentication("superUser");
+
+        @Cleanup
+        PulsarAdmin admin = spy(
+                PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()).authentication(adminAuthentication).build());
+
+        Authentication authentication = new ClientAuthentication(clientRole);
+
+        replacePulsarClient(PulsarClient.builder()
+                .serviceUrl(pulsar.getBrokerServiceUrl())
+                .authentication(authentication));
+
+        admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build());
+
+        admin.tenants().createTenant("public",
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test")));
+        admin.namespaces().createNamespace("public/default", Sets.newHashSet("test"));
+
+        AuthorizationService authorizationService = new AuthorizationService(conf, pulsar.getPulsarResources());
+        TopicName topicName = TopicName.get("persistent://public/default/t1");
+        NamespaceName namespaceName = NamespaceName.get("public/default");
+        String role = "test-role";
+        Set actions = Sets.newHashSet(AuthAction.produce, AuthAction.consume);
+        Assert.assertFalse(authorizationService.canProduce(topicName, role, null));
+        Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1"));
+        authorizationService.grantPermissionAsync(topicName, actions, role, "auth-json").get();
+        Assert.assertTrue(authorizationService.canProduce(topicName, role, null));
+        Assert.assertTrue(authorizationService.canConsume(topicName, role, null, "sub1"));
+
+        authorizationService.revokePermissionAsync(topicName, role).get();
+        Assert.assertFalse(authorizationService.canProduce(topicName, role, null));
+        Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1"));
+
+        authorizationService.grantPermissionAsync(namespaceName, actions, role, null).get();
+        Assert.assertTrue(authorizationService.allowNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPIC, role, null).get());
+        authorizationService.revokePermissionAsync(namespaceName, role).get();
+        Assert.assertFalse(authorizationService.allowNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPIC, role, null).get());
+
+        log.info("-- Exiting {} test --", methodName);
+    }
+
     @Test
     public void testAuthData() throws Exception {
         log.info("-- Starting {} test --", methodName);

From ca00a83c13bf4ac8aab962b6e79a550a55f40bf5 Mon Sep 17 00:00:00 2001
From: Eric Shen 
Date: Fri, 2 Jun 2023 20:17:45 +0800
Subject: [PATCH 364/404] [improve][build] add `--no-cache-dir` in dockerfile
 to disable the pip cache (#20477)

---
 build/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index 7660325567748..89b8f25cd06b2 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -80,7 +80,7 @@ RUN dpkg -i crowdin.deb
 
 # Install PIP
 RUN curl https://bootstrap.pypa.io/get-pip.py  | python3 -
-RUN pip3 install pdoc
+RUN pip3 --no-cache-dir install pdoc
 #
 # Installation
 ARG MAVEN_VERSION=3.6.3

From 43fcf39ea6db3f157ab3c65a607504fa2e4cda15 Mon Sep 17 00:00:00 2001
From: Massimiliano Mirelli 
Date: Sat, 3 Jun 2023 01:44:54 +0200
Subject: [PATCH 365/404] [improve][test] Implement broker integration test
 with entry metadata interceptors enabled (#20398)

---
 ...tallWithEntryMetadataInterceptorsTest.java | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java

diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java
new file mode 100644
index 0000000000000..dd7a7ccff78f6
--- /dev/null
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.integration.bookkeeper;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.tests.integration.docker.ContainerExecResult;
+import org.apache.pulsar.tests.integration.topologies.PulsarCluster;
+import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec;
+import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.joining;
+import static org.testng.AssertJUnit.assertEquals;
+
+/***
+ * Test that verifies that regression in BookKeeper 4.16.0 is fixed.
+ *
+ * Anti-regression test for issue https://github.com/apache/pulsar/issues/20091.
+ */
+@Slf4j
+public class BrokerInstallWithEntryMetadataInterceptorsTest extends PulsarClusterTestBase {
+    private static final String PREFIX = "PULSAR_PREFIX_";
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    public final void setupCluster() throws Exception {
+        incrementSetupNumber();
+
+        final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5))
+                .filter(s -> !s.isEmpty())
+                .collect(joining("-"));
+        brokerEnvs.put(PREFIX + "exposingBrokerEntryMetadataToClientEnabled", "true");
+        brokerEnvs.put(PREFIX + "brokerEntryMetadataInterceptors", "org.apache.pulsar.common.intercept.AppendBrokerTimestampMetadataInterceptor");
+        PulsarClusterSpec spec = PulsarClusterSpec.builder()
+                .numBookies(2)
+                .numBrokers(1)
+                .brokerEnvs(brokerEnvs)
+                .clusterName(clusterName)
+                .build();
+
+        log.info("Setting up cluster {} with {} bookies, {} brokers",
+                spec.clusterName(), spec.numBookies(), spec.numBrokers());
+
+        pulsarCluster = PulsarCluster.forSpec(spec);
+        pulsarCluster.start();
+
+        log.info("Cluster {} is setup", spec.clusterName());
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    public final void tearDownCluster() throws Exception {
+        super.tearDownCluster();
+    }
+
+    @Test
+    public void testBrokerHttpServerIsRunning() throws Exception {
+        ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd(
+                PulsarCluster.CURL,
+                "-X",
+                "GET",
+                "http://localhost:8080/admin/v2/brokers/health");
+        assertEquals(result.getExitCode(), 0);
+        assertEquals("ok", result.getStdout());
+    }
+}

From ceed19cf3b8536a8c9059bfbcb29ef972841b412 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Fri, 2 Jun 2023 18:47:53 -0500
Subject: [PATCH 366/404] [fix][fn] Support customizing TLS config for function
 download command (#20482)

---
 .../runtime/kubernetes/KubernetesRuntime.java | 10 ++++++
 .../kubernetes/KubernetesRuntimeTest.java     | 33 +++++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
index aa474aad801dc..8d7a95cf3d71a 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
@@ -885,6 +885,16 @@ && isNotBlank(authConfig.getClientAuthenticationParameters())) {
                         "--auth-params",
                         authConfig.getClientAuthenticationParameters()));
             }
+            cmd.addAll(Arrays.asList(
+                    "--tls-allow-insecure",
+                    Boolean.toString(authConfig.isTlsAllowInsecureConnection()),
+                    "--tls-enable-hostname-verification",
+                    Boolean.toString(authConfig.isTlsHostnameVerificationEnable())));
+            if (isNotBlank(authConfig.getTlsTrustCertsFilePath())) {
+                cmd.addAll(Arrays.asList(
+                        "--tls-trust-cert-path",
+                        authConfig.getTlsTrustCertsFilePath()));
+            }
         }
 
         cmd.addAll(Arrays.asList(
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index 3facd37fc9244..d6135737c4f21 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -852,6 +852,7 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception {
         V1StatefulSet spec = container.createStatefulSet();
         String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
                 + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
+                + " --tls-allow-insecure false --tls-enable-hostname-verification false"
                 + " functions download "
                 + "--tenant " + TEST_TENANT
                 + " --namespace " + TEST_NAMESPACE
@@ -878,6 +879,38 @@ public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws
         V1StatefulSet spec = container.createStatefulSet();
         String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
                 + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
+                + " --tls-allow-insecure false --tls-enable-hostname-verification false"
+                + " functions download "
+                + "--tenant " + TEST_TENANT
+                + " --namespace " + TEST_NAMESPACE
+                + " --name " + TEST_NAME
+                + " --destination-file " + pulsarRootDir + "/" + userJarFile;
+        String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2);
+        assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand);
+    }
+
+    @Test
+    public void testCustomKubernetesDownloadCommandsWithAuthAndCustomTLSWithoutAuthSpec() throws Exception {
+        InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false);
+        config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false));
+
+        factory = createKubernetesRuntimeFactory(null,
+                10, 1.0, 1.0, Optional.empty(), null, wconfig -> {
+                    wconfig.setAuthenticationEnabled(true);
+                }, AuthenticationConfig.builder()
+                        .clientAuthenticationPlugin("com.MyAuth")
+                        .clientAuthenticationParameters("{\"authParam1\": \"authParamValue1\"}")
+                        .useTls(true) // set to verify it is ignored because pulsar admin does not consider this setting
+                        .tlsHostnameVerificationEnable(true)
+                        .tlsTrustCertsFilePath("/my/ca.pem")
+                        .build());
+
+        KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l);
+        V1StatefulSet spec = container.createStatefulSet();
+        String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
+                + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
+                + " --tls-allow-insecure false --tls-enable-hostname-verification true"
+                + " --tls-trust-cert-path /my/ca.pem"
                 + " functions download "
                 + "--tenant " + TEST_TENANT
                 + " --namespace " + TEST_NAMESPACE

From 8b3c085e9d26605cca544068e8611d72e7012468 Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Fri, 2 Jun 2023 19:51:19 -0400
Subject: [PATCH 367/404] [fix][fn] enable Go function token auth and TLS
 (#20468)

Co-authored-by: Andy Walker 
---
 pulsar-function-go/conf/conf.go               |   8 +-
 pulsar-function-go/examples/go.mod            |   2 -
 pulsar-function-go/examples/go.sum            | 121 ++++++------------
 pulsar-function-go/pf/instance.go             |  37 +++++-
 pulsar-function-go/pf/instanceConf.go         |  10 ++
 .../instance/go/GoInstanceConfig.java         |   7 +
 .../functions/runtime/RuntimeUtils.java       |  21 ++-
 .../functions/runtime/RuntimeUtilsTest.java   |  14 +-
 8 files changed, 130 insertions(+), 90 deletions(-)

diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go
index d52b886b540f9..639da71d25a6d 100644
--- a/pulsar-function-go/conf/conf.go
+++ b/pulsar-function-go/conf/conf.go
@@ -49,7 +49,13 @@ type Conf struct {
 	ProcessingGuarantees int32  `json:"processingGuarantees" yaml:"processingGuarantees"`
 	SecretsMap           string `json:"secretsMap" yaml:"secretsMap"`
 	Runtime              int32  `json:"runtime" yaml:"runtime"`
-	//Deprecated
+	// Authentication
+	ClientAuthenticationPlugin     string `json:"clientAuthenticationPlugin" yaml:"clientAuthenticationPlugin"`
+	ClientAuthenticationParameters string `json:"clientAuthenticationParameters" yaml:"clientAuthenticationParameters"`
+	TLSTrustCertsFilePath          string `json:"tlsTrustCertsFilePath" yaml:"tlsTrustCertsFilePath"`
+	TLSAllowInsecureConnection     bool   `json:"tlsAllowInsecureConnection" yaml:"tlsAllowInsecureConnection"`
+	TLSHostnameVerificationEnable  bool   `json:"tlsHostnameVerificationEnable" yaml:"tlsHostnameVerificationEnable"`
+	// Deprecated
 	AutoACK     bool  `json:"autoAck" yaml:"autoAck"`
 	Parallelism int32 `json:"parallelism" yaml:"parallelism"`
 	//source config
diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod
index 074b50d2e66e9..2b3f1b0c478ce 100644
--- a/pulsar-function-go/examples/go.mod
+++ b/pulsar-function-go/examples/go.mod
@@ -5,8 +5,6 @@ go 1.13
 require (
 	github.com/apache/pulsar-client-go v0.8.1
 	github.com/apache/pulsar/pulsar-function-go v0.0.0
-	github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32 // indirect
-	github.com/yahoo/athenz v1.8.55 // indirect
 )
 
 replace github.com/apache/pulsar/pulsar-function-go => ../
diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum
index d167adb92fe41..0c88121d42346 100644
--- a/pulsar-function-go/examples/go.sum
+++ b/pulsar-function-go/examples/go.sum
@@ -37,18 +37,13 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4=
-github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0=
+github.com/99designs/keyring v1.1.6 h1:kVDC2uCgVwecxCk+9zoCt2uEL6dt+dfVzMvGgnVcIuM=
 github.com/99designs/keyring v1.1.6/go.mod h1:16e0ds7LGQQcT59QqkTg72Hh5ShM51Byv5PEmW6uoRU=
-github.com/AthenZ/athenz v1.10.15 h1:8Bc2W313k/ev/SGokuthNbzpwfg9W3frg3PKq1r943I=
-github.com/AthenZ/athenz v1.10.15/go.mod h1:7KMpEuJ9E4+vMCMI3UQJxwWs0RZtQq7YXZ1IteUjdsc=
+github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY=
 github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/zstd v1.4.6-0.20200617134701-89f69fb7df32 h1:/gZKpgSMydtrih81nvUhlkXpZIUfthKShSCVbRzBt9Y=
-github.com/DataDog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
-github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8=
-github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
 github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -56,24 +51,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/apache/pulsar-client-go v0.0.0-20200113085434-9b739cf9d098/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4=
-github.com/apache/pulsar-client-go v0.0.0-20200116214305-4d788d9935ed h1:Lp7eU5ym84jPmIXoonoaJWVN6psyB90Olookp61LCeA=
-github.com/apache/pulsar-client-go v0.0.0-20200116214305-4d788d9935ed/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4=
-github.com/apache/pulsar-client-go v0.1.0/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4=
-github.com/apache/pulsar-client-go v0.2.0 h1:7teu0FaXzzKPjDdUNjA7dVYKFjCy6OVX5as6nUww4qk=
-github.com/apache/pulsar-client-go v0.2.0/go.mod h1:POSPPmXv1RuoM7FzHaS3NurCSOopwin2ekGK2PcOgVM=
-github.com/apache/pulsar-client-go v0.3.1-0.20201201083639-154bff0bb825 h1:RfvcnGzo67yEHA+eDjoeAEwx5ZxWDgIoMGHJ/Z6Zq9A=
-github.com/apache/pulsar-client-go v0.3.1-0.20201201083639-154bff0bb825/go.mod h1:pTmScVVHRhbB8wh0J+m5ZzHI0Lyfe0TwfPEbYEh+JUw=
-github.com/apache/pulsar-client-go v0.7.0 h1:sZBkjJPHC7akM8n8DuzkLdwioKPSzyub3efCJ1Ltw9Y=
-github.com/apache/pulsar-client-go v0.7.0/go.mod h1:EauTUv9sTmP9QRznRgK9hxnzCsIVfS8fyhTfGcuJBrE=
 github.com/apache/pulsar-client-go v0.8.1 h1:UZINLbH3I5YtNzqkju7g9vrl4CKrEgYSx2rbpvGufrE=
 github.com/apache/pulsar-client-go v0.8.1/go.mod h1:yJNcvn/IurarFDxwmoZvb2Ieylg630ifxeO/iXpk27I=
-github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb h1:E1P0FudxDdj2RhbveZC9i3PwukLCA/4XQSkBS/dw6/I=
-github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY=
-github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd h1:P5kM7jcXJ7TaftX0/EMKiSJgvQc/ct+Fw0KMvcH3WuY=
-github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY=
+github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e h1:EqiJ0Xil8NmcXyupNqXV9oYDBeWntEIegxLahrTr8DY=
 github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e/go.mod h1:Xee4tgYLFpYcPMcTfBYWE1uKRzeciodGTSEDMzsR6i8=
-github.com/apache/pulsar/pulsar-function-go v0.0.0-20200124033432-ec122ed9562c/go.mod h1:2a3PacwSg4KPcGxO3bjH29xsoKSuSkq2mG0sjKtxsP4=
 github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
 github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
 github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
@@ -81,7 +62,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
 github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -90,7 +70,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
-github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -104,16 +83,14 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU=
 github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U=
-github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:inRp+etsHuvVqMPNTXaFlpf/Tj7wqviBtdJoPVrPEFQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
 github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
-github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU=
-github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
+github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b h1:HBah4D48ypg3J7Np4N+HY/ZR76fx3HEUGxDU6Uk39oQ=
 github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -124,6 +101,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -137,14 +115,14 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -160,7 +138,6 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
@@ -171,12 +148,11 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -192,6 +168,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -209,14 +186,14 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
@@ -243,12 +220,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk=
-github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
 github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -258,20 +233,20 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d h1:Z+RDyXzjKE0i2sTjZ/b1uxiGtPhFy34Ou/Tk0qwN0kM=
 github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
-github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY=
 github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg=
 github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
@@ -299,11 +274,14 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -311,7 +289,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -321,26 +298,22 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
-github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
-github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
-github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -349,9 +322,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -359,25 +331,22 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/yahoo/athenz v1.8.55 h1:xGhxN3yLq334APyn0Zvcc+aqu78Q7BBhYJevM3EtTW0=
-github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -402,7 +371,6 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -451,7 +419,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -471,14 +438,11 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -487,12 +451,12 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
@@ -500,6 +464,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc=
 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -521,7 +486,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -529,13 +493,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -551,12 +513,10 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -571,24 +531,22 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -601,7 +559,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -643,6 +600,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -672,6 +630,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -679,7 +638,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@@ -699,7 +657,6 @@ google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -715,15 +672,14 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
@@ -737,6 +693,7 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -744,36 +701,36 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 2c2deb19b2813..1064aece46fe8 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -21,8 +21,10 @@ package pf
 
 import (
 	"context"
+	"fmt"
 	"math"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/golang/protobuf/ptypes/empty"
@@ -192,11 +194,40 @@ CLOSE:
 	return nil
 }
 
+const (
+	authPluginToken = "org.apache.pulsar.client.impl.auth.AuthenticationToken"
+	authPluginNone  = ""
+)
+
 func (gi *goInstance) setupClient() error {
-	client, err := pulsar.NewClient(pulsar.ClientOptions{
+	ic := gi.context.instanceConf
+
+	clientOpts := pulsar.ClientOptions{
+		URL:                        ic.pulsarServiceURL,
+		TLSTrustCertsFilePath:      ic.tlsTrustCertsPath,
+		TLSAllowInsecureConnection: ic.tlsAllowInsecure,
+		TLSValidateHostname:        ic.tlsHostnameVerification,
+	}
+
+	switch ic.authPlugin {
+	case authPluginToken:
+		switch {
+		case strings.HasPrefix(ic.authParams, "file://"):
+			clientOpts.Authentication = pulsar.NewAuthenticationTokenFromFile(ic.authParams[7:])
+		case strings.HasPrefix(ic.authParams, "token:"):
+			clientOpts.Authentication = pulsar.NewAuthenticationToken(ic.authParams[6:])
+		case ic.authParams == "":
+			return fmt.Errorf("auth plugin %s given, but authParams is empty", authPluginToken)
+		default:
+			return fmt.Errorf(`unknown token format - expecting "file://" or "token:" prefix`)
+		}
+	case authPluginNone:
+		clientOpts.Authentication, _ = pulsar.NewAuthentication("", "") // ret: auth.NewAuthDisabled()
+	default:
+		return fmt.Errorf("unknown auth provider: %s", ic.authPlugin)
+	}
 
-		URL: gi.context.instanceConf.pulsarServiceURL,
-	})
+	client, err := pulsar.NewClient(clientOpts)
 	if err != nil {
 		log.Errorf("create client error:%v", err)
 		gi.stats.incrTotalSysExceptions(err)
diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go
index 9d4cabfae5a9b..72bcaf9bbb2c0 100644
--- a/pulsar-function-go/pf/instanceConf.go
+++ b/pulsar-function-go/pf/instanceConf.go
@@ -42,6 +42,11 @@ type instanceConf struct {
 	killAfterIdle               time.Duration
 	expectedHealthCheckInterval int32
 	metricsPort                 int
+	authPlugin                  string
+	authParams                  string
+	tlsTrustCertsPath           string
+	tlsAllowInsecure            bool
+	tlsHostnameVerification     bool
 }
 
 func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf {
@@ -107,6 +112,11 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf {
 			},
 			UserConfig: cfg.UserConfig,
 		},
+		authPlugin:              cfg.ClientAuthenticationPlugin,
+		authParams:              cfg.ClientAuthenticationParameters,
+		tlsTrustCertsPath:       cfg.TLSTrustCertsFilePath,
+		tlsAllowInsecure:        cfg.TLSAllowInsecureConnection,
+		tlsHostnameVerification: cfg.TLSHostnameVerificationEnable,
 	}
 
 	if instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_EFFECTIVELY_ONCE {
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
index 67fe2a41d553d..bcbc7fa057ff9 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
@@ -44,6 +44,13 @@ public class GoInstanceConfig {
     private int processingGuarantees;
     private String secretsMap = "";
     private String userConfig = "";
+
+    private String clientAuthenticationPlugin = "";
+    private String clientAuthenticationParameters = "";
+    private String tlsTrustCertsFilePath = "";
+    private boolean tlsHostnameVerificationEnable = false;
+    private boolean tlsAllowInsecureConnection = false;
+
     private int runtime;
     private boolean autoAck;
     private int parallelism;
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
index 1d2eacbd77b12..1b6b694616375 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
@@ -129,6 +129,7 @@ public static List getArgsBeforeCmd(InstanceConfig instanceConfig, Strin
      */
 
     public static List getGoInstanceCmd(InstanceConfig instanceConfig,
+                                                AuthenticationConfig authConfig,
                                                 String originalCodeFileName,
                                                 String pulsarServiceUrl,
                                                 boolean k8sRuntime) throws IOException {
@@ -187,6 +188,23 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig,
             goInstanceConfig.setParallelism(instanceConfig.getFunctionDetails().getParallelism());
         }
 
+        if (authConfig != null) {
+            if (isNotBlank(authConfig.getClientAuthenticationPlugin())
+                    && isNotBlank(authConfig.getClientAuthenticationParameters())) {
+                goInstanceConfig.setClientAuthenticationPlugin(authConfig.getClientAuthenticationPlugin());
+                goInstanceConfig.setClientAuthenticationParameters(authConfig.getClientAuthenticationParameters());
+            }
+            goInstanceConfig.setTlsAllowInsecureConnection(
+                    authConfig.isTlsAllowInsecureConnection());
+            goInstanceConfig.setTlsHostnameVerificationEnable(
+                    authConfig.isTlsHostnameVerificationEnable());
+            if (isNotBlank(authConfig.getTlsTrustCertsFilePath())){
+                goInstanceConfig.setTlsTrustCertsFilePath(
+                        authConfig.getTlsTrustCertsFilePath());
+            }
+
+        }
+
         if (instanceConfig.getMaxBufferedTuples() != 0) {
             goInstanceConfig.setMaxBufTuples(instanceConfig.getMaxBufferedTuples());
         }
@@ -292,7 +310,8 @@ public static List getCmd(InstanceConfig instanceConfig,
         final List args = new LinkedList<>();
 
         if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.GO) {
-            return getGoInstanceCmd(instanceConfig, originalCodeFileName, pulsarServiceUrl, k8sRuntime);
+            return getGoInstanceCmd(instanceConfig, authConfig,
+                    originalCodeFileName, pulsarServiceUrl, k8sRuntime);
         }
 
         if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) {
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
index 27c2f2618278f..a0d11b551f322 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
@@ -73,6 +73,13 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException {
         instanceConfig.setPort(1337);
         instanceConfig.setMetricsPort(60000);
 
+        AuthenticationConfig authConfig = AuthenticationConfig.builder()
+                .clientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.AuthenticationToken")
+                .clientAuthenticationParameters("file:///secret/token.jwt")
+                .tlsTrustCertsFilePath("/secret/ca.cert.pem")
+                .tlsHostnameVerificationEnable(true)
+                .tlsAllowInsecureConnection(false)
+                .build();
 
         JSONObject userConfig = new JSONObject();
         userConfig.put("word-of-the-day", "der Weltschmerz");
@@ -116,7 +123,7 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException {
 
         instanceConfig.setFunctionDetails(functionDetails);
 
-        List commands = RuntimeUtils.getGoInstanceCmd(instanceConfig, "config", "pulsar://localhost:6650", k8sRuntime);
+        List commands = RuntimeUtils.getGoInstanceCmd(instanceConfig, authConfig,"config", "pulsar://localhost:6650", k8sRuntime);
         if (k8sRuntime) {
             goInstanceConfig = new ObjectMapper().readValue(commands.get(2).replaceAll("^\'|\'$", ""), HashMap.class);
         } else {
@@ -160,6 +167,11 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException {
         Assert.assertEquals(goInstanceConfig.get("deadLetterTopic"), "go-func-deadletter");
         Assert.assertEquals(goInstanceConfig.get("userConfig"), userConfig.toString());
         Assert.assertEquals(goInstanceConfig.get("metricsPort"), 60000);
+        Assert.assertEquals(goInstanceConfig.get("clientAuthenticationPlugin"), "org.apache.pulsar.client.impl.auth.AuthenticationToken");
+        Assert.assertEquals(goInstanceConfig.get("clientAuthenticationParameters"), "file:///secret/token.jwt");
+        Assert.assertEquals(goInstanceConfig.get("tlsTrustCertsFilePath"), "/secret/ca.cert.pem");
+        Assert.assertEquals(goInstanceConfig.get("tlsHostnameVerificationEnable"), true);
+        Assert.assertEquals(goInstanceConfig.get("tlsAllowInsecureConnection"), false);
     }
 
     @DataProvider(name = "k8sRuntime")

From 510744c20d0339c4b29366f07bf1849cabb81dec Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Sat, 3 Jun 2023 03:01:18 -0400
Subject: [PATCH 368/404] [feat][fn] Add stateStorageURL and pulsarWebService
 URL to go InstanceConfig (#20443)

Co-authored-by: Andy Walker 
---
 pulsar-function-go/conf/conf.go                | 18 ++++++++++--------
 pulsar-function-go/pf/instanceConf.go          |  4 ++++
 .../instance/go/GoInstanceConfig.java          |  2 ++
 .../pulsar/functions/runtime/RuntimeUtils.java | 15 +++++++++++++--
 .../functions/runtime/RuntimeUtilsTest.java    |  5 ++++-
 5 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go
index 639da71d25a6d..03513648fac3b 100644
--- a/pulsar-function-go/conf/conf.go
+++ b/pulsar-function-go/conf/conf.go
@@ -33,14 +33,16 @@ import (
 const ConfigPath = "conf/conf.yaml"
 
 type Conf struct {
-	PulsarServiceURL string        `json:"pulsarServiceURL" yaml:"pulsarServiceURL"`
-	InstanceID       int           `json:"instanceID" yaml:"instanceID"`
-	FuncID           string        `json:"funcID" yaml:"funcID"`
-	FuncVersion      string        `json:"funcVersion" yaml:"funcVersion"`
-	MaxBufTuples     int           `json:"maxBufTuples" yaml:"maxBufTuples"`
-	Port             int           `json:"port" yaml:"port"`
-	ClusterName      string        `json:"clusterName" yaml:"clusterName"`
-	KillAfterIdleMs  time.Duration `json:"killAfterIdleMs" yaml:"killAfterIdleMs"`
+	PulsarServiceURL       string        `json:"pulsarServiceURL" yaml:"pulsarServiceURL"`
+	StateStorageServiceURL string        `json:"stateStorageServiceUrl" yaml:"stateStorageServiceUrl"`
+	PulsarWebServiceURL    string        `json:"pulsarWebServiceUrl" yaml:"pulsarWebServiceUrl"`
+	InstanceID             int           `json:"instanceID" yaml:"instanceID"`
+	FuncID                 string        `json:"funcID" yaml:"funcID"`
+	FuncVersion            string        `json:"funcVersion" yaml:"funcVersion"`
+	MaxBufTuples           int           `json:"maxBufTuples" yaml:"maxBufTuples"`
+	Port                   int           `json:"port" yaml:"port"`
+	ClusterName            string        `json:"clusterName" yaml:"clusterName"`
+	KillAfterIdleMs        time.Duration `json:"killAfterIdleMs" yaml:"killAfterIdleMs"`
 	// function details config
 	Tenant               string `json:"tenant" yaml:"tenant"`
 	NameSpace            string `json:"nameSpace" yaml:"nameSpace"`
diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go
index 72bcaf9bbb2c0..4cb60dd258ad9 100644
--- a/pulsar-function-go/pf/instanceConf.go
+++ b/pulsar-function-go/pf/instanceConf.go
@@ -39,6 +39,8 @@ type instanceConf struct {
 	port                        int
 	clusterName                 string
 	pulsarServiceURL            string
+	stateServiceURL             string
+	pulsarWebServiceURL         string
 	killAfterIdle               time.Duration
 	expectedHealthCheckInterval int32
 	metricsPort                 int
@@ -76,6 +78,8 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf {
 		port:                        cfg.Port,
 		clusterName:                 cfg.ClusterName,
 		pulsarServiceURL:            cfg.PulsarServiceURL,
+		stateServiceURL:             cfg.StateStorageServiceURL,
+		pulsarWebServiceURL:         cfg.PulsarWebServiceURL,
 		killAfterIdle:               cfg.KillAfterIdleMs,
 		expectedHealthCheckInterval: cfg.ExpectedHealthCheckInterval,
 		metricsPort:                 cfg.MetricsPort,
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
index bcbc7fa057ff9..599b6ed8f4fdf 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java
@@ -27,6 +27,8 @@
 @Getter
 public class GoInstanceConfig {
     private String pulsarServiceURL = "";
+    private String stateStorageServiceUrl = "";
+    private String pulsarWebServiceUrl = "";
     private int instanceID;
     private String funcID = "";
     private String funcVersion = "";
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
index 1b6b694616375..67469041c949c 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
@@ -132,6 +132,8 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig,
                                                 AuthenticationConfig authConfig,
                                                 String originalCodeFileName,
                                                 String pulsarServiceUrl,
+                                                String stateStorageServiceUrl,
+                                                String pulsarWebServiceUrl,
                                                 boolean k8sRuntime) throws IOException {
         final List args = new LinkedList<>();
         GoInstanceConfig goInstanceConfig = new GoInstanceConfig();
@@ -140,6 +142,14 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig,
             goInstanceConfig.setClusterName(instanceConfig.getClusterName());
         }
 
+        if (null != stateStorageServiceUrl) {
+            goInstanceConfig.setStateStorageServiceUrl(stateStorageServiceUrl);
+        }
+
+        if (instanceConfig.isExposePulsarAdminClientEnabled() && StringUtils.isNotBlank(pulsarWebServiceUrl)) {
+            goInstanceConfig.setPulsarWebServiceUrl(pulsarWebServiceUrl);
+        }
+
         if (instanceConfig.getInstanceId() != 0) {
             goInstanceConfig.setInstanceID(instanceConfig.getInstanceId());
         }
@@ -310,8 +320,9 @@ public static List getCmd(InstanceConfig instanceConfig,
         final List args = new LinkedList<>();
 
         if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.GO) {
-            return getGoInstanceCmd(instanceConfig, authConfig,
-                    originalCodeFileName, pulsarServiceUrl, k8sRuntime);
+            return getGoInstanceCmd(instanceConfig, authConfig, originalCodeFileName,
+                    pulsarServiceUrl, stateStorageServiceUrl, pulsarWebServiceUrl,
+                    k8sRuntime);
         }
 
         if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) {
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
index a0d11b551f322..a9dfcd7ffc777 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
@@ -122,8 +122,9 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException {
                 .build();
 
         instanceConfig.setFunctionDetails(functionDetails);
+        instanceConfig.setExposePulsarAdminClientEnabled(true);
 
-        List commands = RuntimeUtils.getGoInstanceCmd(instanceConfig, authConfig,"config", "pulsar://localhost:6650", k8sRuntime);
+        List commands = RuntimeUtils.getGoInstanceCmd(instanceConfig, authConfig, "config", "pulsar://localhost:6650", "bk://localhost:4181",  "http://localhost:8080", k8sRuntime);
         if (k8sRuntime) {
             goInstanceConfig = new ObjectMapper().readValue(commands.get(2).replaceAll("^\'|\'$", ""), HashMap.class);
         } else {
@@ -151,6 +152,8 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException {
         Assert.assertEquals(goInstanceConfig.get("autoAck"), true);
         Assert.assertEquals(goInstanceConfig.get("regexPatternSubscription"), false);
         Assert.assertEquals(goInstanceConfig.get("pulsarServiceURL"), "pulsar://localhost:6650");
+        Assert.assertEquals(goInstanceConfig.get("stateStorageServiceUrl"), "bk://localhost:4181");
+        Assert.assertEquals(goInstanceConfig.get("pulsarWebServiceUrl"), "http://localhost:8080");
         Assert.assertEquals(goInstanceConfig.get("runtime"), 3);
         Assert.assertEquals(goInstanceConfig.get("cpu"), 2.0);
         Assert.assertEquals(goInstanceConfig.get("funcID"), "func-7734");

From 43b3622cc7e7f746ca9920fdc704dc7448767ac7 Mon Sep 17 00:00:00 2001
From: Tao Jiuming <95597048+tjiuming@users.noreply.github.com>
Date: Sat, 3 Jun 2023 21:14:16 +0800
Subject: [PATCH 369/404] [improve][broker] Add `topic_load_failed` metric
 (#19236)

Co-authored-by: daojun 
---
 .../pulsar/broker/service/BrokerService.java  |  9 +++
 .../pulsar/broker/service/PulsarStats.java    |  4 +
 .../stats/BrokerOperabilityMetrics.java       | 10 ++-
 .../broker/service/BrokerServiceTest.java     | 80 +++++++++++++++++++
 .../broker/stats/PrometheusMetricsTest.java   |  4 +
 5 files changed, 106 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 663d013dc7439..3dc7718c32a74 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1226,6 +1226,10 @@ public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture> createNonPersistentTopic(String topic) {
         CompletableFuture> topicFuture = new CompletableFuture<>();
+        topicFuture.exceptionally(t -> {
+            pulsarStats.recordTopicLoadFailed();
+            return null;
+        });
         if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) {
             if (log.isDebugEnabled()) {
                 log.debug("Broker is unable to load non-persistent topic {}", topic);
@@ -1618,6 +1622,11 @@ private void createPersistentTopic(final String topic, boolean createIfMissing,
         TopicName topicName = TopicName.get(topic);
         final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
 
+        topicFuture.exceptionally(t -> {
+            pulsarStats.recordTopicLoadFailed();
+            return null;
+        });
+
         if (isTransactionInternalName(topicName)) {
             String msg = String.format("Can not create transaction system topic %s", topic);
             log.warn(msg);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
index e959e9bbda2bb..db14892d26663 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
@@ -265,6 +265,10 @@ public void recordTopicLoadTimeValue(String topic, long topicLoadLatencyMs) {
         }
     }
 
+    public void recordTopicLoadFailed() {
+        brokerOperabilityMetrics.recordTopicLoadFailed();
+    }
+
     public void recordConnectionCreate() {
         brokerOperabilityMetrics.recordConnectionCreate();
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
index 909133338719f..400dbd3335a2a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.stats;
 
+import io.prometheus.client.Counter;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -29,6 +30,7 @@
 /**
  */
 public class BrokerOperabilityMetrics {
+    private static final Counter TOPIC_LOAD_FAILED = Counter.build("topic_load_failed", "-").register();
     private final List metricsList;
     private final String localCluster;
     private final DimensionStats topicLoadStats;
@@ -84,7 +86,9 @@ Map getDimensionMap(String metricsName) {
     }
 
     Metrics getTopicLoadMetrics() {
-        return getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats);
+        Metrics metrics = getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats);
+        metrics.put("brk_topic_load_failed_count", TOPIC_LOAD_FAILED.get());
+        return metrics;
     }
 
     Metrics getDimensionMetrics(String metricsName, String dimensionName, DimensionStats stats) {
@@ -112,6 +116,10 @@ public void recordTopicLoadTimeValue(long topicLoadLatencyMs) {
         topicLoadStats.recordDimensionTimeValue(topicLoadLatencyMs, TimeUnit.MILLISECONDS);
     }
 
+    public void recordTopicLoadFailed() {
+        this.TOPIC_LOAD_FAILED.inc();
+    }
+
     public void recordConnectionCreate() {
         this.connectionTotalCreatedCount.increment();
         this.connectionActive.increment();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index e9b7ddb991e57..1d45c87b2f256 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -33,6 +33,7 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.EventLoopGroup;
 import io.netty.util.concurrent.DefaultThreadFactory;
@@ -107,6 +108,7 @@
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
 import org.awaitility.Awaitility;
+import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -1513,4 +1515,82 @@ public void testDynamicConfigurationsForceDeleteTenantAllowed() throws Exception
             assertTrue(conf.isForceDeleteTenantAllowed());
         });
     }
+
+
+    @Test
+    public void testBrokerStatsTopicLoadFailed() throws Exception {
+        admin.namespaces().createNamespace("prop/ns-test");
+
+        String persistentTopic = "persistent://prop/ns-test/topic1_" + UUID.randomUUID();
+        String nonPersistentTopic = "non-persistent://prop/ns-test/topic2_" + UUID.randomUUID();
+
+        BrokerService brokerService = pulsar.getBrokerService();
+        brokerService = Mockito.spy(brokerService);
+        // mock create persistent topic failed
+        Mockito
+                .doAnswer(invocation -> {
+                    CompletableFuture f = new CompletableFuture<>();
+                    f.completeExceptionally(new RuntimeException("This is an exception"));
+                    return f;
+                })
+                .when(brokerService).getManagedLedgerConfig(Mockito.eq(TopicName.get(persistentTopic)));
+
+        // mock create non-persistent topic failed
+        Mockito
+                .doAnswer(inv -> {
+                    CompletableFuture f = new CompletableFuture<>();
+                    f.completeExceptionally(new RuntimeException("This is an exception"));
+                    return f;
+                })
+                .when(brokerService).checkTopicNsOwnership(Mockito.eq(nonPersistentTopic));
+
+
+        PulsarService pulsarService = pulsar;
+        Field field = PulsarService.class.getDeclaredField("brokerService");
+        field.setAccessible(true);
+        field.set(pulsarService, brokerService);
+
+        CompletableFuture> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(persistentTopic)
+                .createAsync();
+        CompletableFuture> producer1 = pulsarClient.newProducer(Schema.STRING)
+                .topic(nonPersistentTopic)
+                .createAsync();
+
+        producer.whenComplete((v, t) -> {
+            if (t == null) {
+                try {
+                    v.close();
+                } catch (PulsarClientException e) {
+                    // ignore
+                }
+            }
+        });
+        producer1.whenComplete((v, t) -> {
+            if (t == null) {
+                try {
+                    v.close();
+                } catch (PulsarClientException e) {
+                    // ignore
+                }
+            }
+        });
+
+        Awaitility.waitAtMost(2, TimeUnit.MINUTES).until(() -> {
+            String json = admin.brokerStats().getMetrics();
+            JsonArray metrics = new Gson().fromJson(json, JsonArray.class);
+            AtomicBoolean flag = new AtomicBoolean(false);
+
+            metrics.forEach(ele -> {
+                JsonObject obj = ((JsonObject) ele);
+                JsonObject metrics0 = (JsonObject) obj.get("metrics");
+                JsonPrimitive v = (JsonPrimitive) metrics0.get("brk_topic_load_failed_count");
+                if (null != v && v.getAsDouble() >= 2D) {
+                    flag.set(true);
+                }
+            });
+
+            return flag.get();
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index a7a28afd8ac64..6cb7378330f09 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -333,6 +333,10 @@ public void testPerTopicStats() throws Exception {
         assertEquals(cm.size(), 1);
         assertEquals(cm.get(0).tags.get("cluster"), "test");
 
+        cm = (List) metrics.get("topic_load_failed_total");
+        assertEquals(cm.size(), 1);
+        assertEquals(cm.get(0).tags.get("cluster"), "test");
+
         cm = (List) metrics.get("pulsar_in_bytes_total");
         assertEquals(cm.size(), 2);
         assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");

From 2dec2677743a6c517deb6976718fbe3a9eef0644 Mon Sep 17 00:00:00 2001
From: crossoverJie 
Date: Mon, 5 Jun 2023 16:10:44 +0800
Subject: [PATCH 370/404] [fix][client] Fix  where the function
 getMsgNumInReceiverQueue always returns 0 when using message listener
 (#20245)

---
 .../api/SimpleProducerConsumerTest.java       | 29 +++++++++++++++++++
 .../pulsar/client/impl/ConsumerBase.java      |  8 +++++
 .../impl/ConsumerStatsRecorderImpl.java       |  6 +++-
 3 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index f3a00531eba56..c45c2b1522f59 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -4109,6 +4109,35 @@ public void testGetStats() throws Exception {
         consumer.close();
         producer.close();
     }
+    @Test(timeOut = 100000)
+    public void testMessageListenerGetStats() throws Exception {
+        final String topicName = "persistent://my-property/my-ns/testGetStats" + UUID.randomUUID();
+        final String subName = "my-sub";
+        final int receiveQueueSize = 100;
+        @Cleanup
+        PulsarClient client = newPulsarClient(lookupUrl.toString(), 100);
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .enableBatching(false).topic(topicName).create();
+        ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING)
+                .messageListener((MessageListener) (consumer1, msg) -> {
+                    try {
+                        TimeUnit.SECONDS.sleep(10);
+                    } catch (InterruptedException igr) {
+                    }
+                })
+                .topic(topicName).receiverQueueSize(receiveQueueSize).subscriptionName(subName).subscribe();
+        Assert.assertNull(consumer.getStats().getMsgNumInSubReceiverQueue());
+        Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), 0);
+
+        for (int i = 0; i < receiveQueueSize; i++) {
+            producer.sendAsync("msg" + i);
+        }
+        //Give some time to consume
+        Awaitility.await()
+                .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), receiveQueueSize));
+        consumer.close();
+        producer.close();
+    }
 
     @Test(timeOut = 100000)
     public void testGetStatsForPartitionedTopic() throws Exception {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
index 0db2a8e0ab9f5..e933005f2d6ea 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
@@ -88,6 +88,11 @@ public abstract class ConsumerBase extends HandlerState implements Consumer>> pendingReceives;
     protected final int maxReceiverQueueSize;
     private volatile int currentReceiverQueueSize;
+
+    protected static final AtomicIntegerFieldUpdater MESSAGE_LISTENER_QUEUE_SIZE_UPDATER =
+            AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "messageListenerQueueSize");
+    protected volatile int messageListenerQueueSize = 0;
+
     protected static final AtomicIntegerFieldUpdater CURRENT_RECEIVER_QUEUE_SIZE_UPDATER =
             AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "currentReceiverQueueSize");
     protected final Schema schema;
@@ -1105,6 +1110,7 @@ private void triggerListener() {
                         // Trigger the notification on the message listener in a separate thread to avoid blocking the
                         // internal pinned executor thread while the message processing happens
                         final Message finalMsg = msg;
+                        MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.incrementAndGet(this);
                         if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) {
                             executorProvider.getExecutor(peekMessageKey(msg)).execute(() ->
                                     callMessageListener(finalMsg));
@@ -1148,6 +1154,8 @@ protected void callMessageListener(Message msg) {
         } catch (Throwable t) {
             log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription,
                     msg.getMessageId(), t);
+        } finally {
+            MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.decrementAndGet(this);
         }
     }
 
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java
index 8630bedc65f31..3a47ddc5d4b08 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java
@@ -238,7 +238,11 @@ public void updateCumulativeStats(ConsumerStats stats) {
     @Override
     public Integer getMsgNumInReceiverQueue() {
         if (consumer instanceof ConsumerBase) {
-            return ((ConsumerBase) consumer).incomingMessages.size();
+            ConsumerBase consumerBase = (ConsumerBase) consumer;
+            if (consumerBase.listener != null){
+                return ConsumerBase.MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.get(consumerBase);
+            }
+            return consumerBase.incomingMessages.size();
         }
         return null;
     }

From 3b862ae614fae795e3312c0b298cc3c2b33a698f Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 5 Jun 2023 14:48:46 +0300
Subject: [PATCH 371/404] [fix][ci] Fix OWASP dependency check suppressions
 (#20486)

---
 pom.xml                                     |  2 +-
 src/owasp-dependency-check-suppressions.xml | 70 +++------------------
 2 files changed, 10 insertions(+), 62 deletions(-)

diff --git a/pom.xml b/pom.xml
index 699fc1631c846..6f1d4d7118608 100644
--- a/pom.xml
+++ b/pom.xml
@@ -297,7 +297,7 @@ flexible messaging model and an intuitive client API.
     0.1.4
     1.3
     0.4
-    8.1.2
+    8.2.1
     0.9.44
     1.6.1
     6.4.0
diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml
index 5a595af2c0a79..18935d0ef2bbf 100644
--- a/src/owasp-dependency-check-suppressions.xml
+++ b/src/owasp-dependency-check-suppressions.xml
@@ -229,72 +229,20 @@
         CVE-2021-42550
     
 
-    
     
-        
-        861af62ae22a71d30f401a80049397fe7ff44423
-        CVE-2020-15113
+        Ignore etdc CVEs in jetcd
+        .*jetcd.*
+        cpe:/a:etcd:etcd
     
-
     
-        
-        663f2ccc0ec7797954c333fa75feeb7d559948b0
-        CVE-2020-15113
+        Ignore etdc CVEs in jetcd
+        .*jetcd.*
+        cpe:/a:redhat:etcd
     
-
     
-        
-        abd0ffcd4e66046057c3bfb34affc0de870a038b
-        CVE-2020-15113
-    
-
-    
-        
-        a2e99802fec5586daca0c4daae975fe601a057a8
-        CVE-2017-8359
-    
-    
-        
-        a2e99802fec5586daca0c4daae975fe601a057a8
-        CVE-2020-15113
-    
-    
-        
-        a2e99802fec5586daca0c4daae975fe601a057a8
-        CVE-2020-7768
-    
-    
-        
-        a2e99802fec5586daca0c4daae975fe601a057a8
-        CVE-2017-7861
-    
-    
-        
-        a2e99802fec5586daca0c4daae975fe601a057a8
-        CVE-2017-9431
-    
-    
-        
-        a2e99802fec5586daca0c4daae975fe601a057a8
-        CVE-2017-7860
+        Ignore grpc CVEs in jetcd
+        .*jetcd-grpc.*
+        cpe:/a:grpc:grpc
     
 
     

From 7e6ca31dcb7bd04afff4daf50ac44be4749b6f2b Mon Sep 17 00:00:00 2001
From: Andy Walker 
Date: Mon, 5 Jun 2023 09:16:51 -0400
Subject: [PATCH 372/404] [fix][fn] Go functions need to use static grpcPort in
 k8s runtime (#20404)

---
 .../runtime/kubernetes/KubernetesRuntimeFactory.java         | 5 +++++
 .../functions/runtime/kubernetes/KubernetesRuntimeTest.java  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
index 3e1d40e80dc6e..bbb6e3992a018 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
@@ -302,6 +302,11 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c
                 .map((customizer) -> customizer.customizeName(instanceConfig.getFunctionDetails(), jobName))
                 .orElse(jobName);
 
+        // pass grpcPort configured in functionRuntimeFactoryConfigs.grpcPort in functions_worker.yml
+        if (grpcPort != null) {
+            instanceConfig.setPort(grpcPort);
+        }
+
         // pass metricsPort configured in functionRuntimeFactoryConfigs.metricsPort in functions_worker.yml
         if (metricsPort != null) {
             instanceConfig.setMetricsPort(metricsPort);
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index d6135737c4f21..2cbbeb8a00b92 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -984,7 +984,7 @@ private void verifyGolangInstance(InstanceConfig config) throws Exception {
         assertEquals(goInstanceConfig.get("disk"), 10000);
         assertEquals(goInstanceConfig.get("instanceID"), 0);
         assertEquals(goInstanceConfig.get("cleanupSubscription"), false);
-        assertEquals(goInstanceConfig.get("port"), 0);
+        assertEquals(goInstanceConfig.get("port"), 4332);
         assertEquals(goInstanceConfig.get("subscriptionType"), 0);
         assertEquals(goInstanceConfig.get("timeoutMs"), 0);
         assertEquals(goInstanceConfig.get("subscriptionName"), "");

From 90f9651306426d82b35f3dec6380001e7b0d208e Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 5 Jun 2023 19:37:07 +0300
Subject: [PATCH 373/404] [fix][test] Disable invalid test
 BrokerServiceTest#testBrokerStatsTopicLoadFailed (#20494)

---
 .../org/apache/pulsar/broker/service/BrokerServiceTest.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index 1d45c87b2f256..d815445b5fa65 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -1517,7 +1517,8 @@ public void testDynamicConfigurationsForceDeleteTenantAllowed() throws Exception
     }
 
 
-    @Test
+    // this test is disabled since it is flaky
+    @Test(enabled = false)
     public void testBrokerStatsTopicLoadFailed() throws Exception {
         admin.namespaces().createNamespace("prop/ns-test");
 

From 4a89f028a3e33c18e2d9efa4e2537ac9f9e82a08 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 5 Jun 2023 23:00:50 +0300
Subject: [PATCH 374/404] [improve][ci] Increase Maven max heap size to 1024m
 in all GHA workflows (#20487)

---
 .github/workflows/ci-go-functions.yaml           | 2 +-
 .github/workflows/ci-maven-cache-update.yaml     | 2 +-
 .github/workflows/ci-owasp-dependency-check.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index f96a6d6586e6e..332ecf0f35229 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -32,7 +32,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   preconditions:
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index 15fefaf3f1645..f20feeea5c9db 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -42,7 +42,7 @@ on:
     - cron: '30 */12 * * *'
 
 env:
-  MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   update-maven-dependencies-cache:
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index 06edbae51adde..090221e699d01 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,7 +24,7 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   run-owasp-dependency-check:

From b1822ed2b4714e4b485ab695eb1d52634ecc8db4 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 5 Jun 2023 17:36:13 -0500
Subject: [PATCH 375/404] [improve][fn] Optimize string concatenation in user
 metrics gen (#20499)

### Motivation

Minor optimization for metrics generation in functions. String concatenation is faster than string format. This is valuable in the context of metrics where we can generate many strings in a tight loop.

### Modifications

* Replace `String.format` with string concatenation in several locations.
* Only create metrics prefix once.

### Verifying this change

This change is a trivial rework / code cleanup without any test coverage.

### Documentation

- [x] `doc-not-needed`
---
 .../org/apache/pulsar/functions/instance/ContextImpl.java  | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
index 5cbbcad24c7a2..d03f57e97205c 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
@@ -603,12 +603,13 @@ public Map getMetrics() {
             String metricName = userMetricsLabelsEntry.getKey();
             String[] labels = userMetricsLabelsEntry.getValue();
             Summary.Child.Value summary = userMetricsSummary.labels(labels).get();
-            metricsMap.put(String.format("%s%s_sum", USER_METRIC_PREFIX, metricName), summary.sum);
-            metricsMap.put(String.format("%s%s_count", USER_METRIC_PREFIX, metricName), summary.count);
+            String prefix = USER_METRIC_PREFIX + metricName + "_";
+            metricsMap.put(prefix + "sum", summary.sum);
+            metricsMap.put(prefix + "count", summary.count);
             for (Map.Entry entry : summary.quantiles.entrySet()) {
                 Double quantile = entry.getKey();
                 Double value = entry.getValue();
-                metricsMap.put(String.format("%s%s_%s", USER_METRIC_PREFIX, metricName, quantile), value);
+                metricsMap.put(prefix + quantile, value);
             }
         }
         return metricsMap;

From 62c6f0872f65a9861c07fb3555d0364ddcd2d0c1 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Tue, 6 Jun 2023 10:26:11 +0800
Subject: [PATCH 376/404] [improve][admin] Using AuthorizationService to revoke
 the permissions. (#20478)

---
 .../PulsarAuthorizationProvider.java          |   9 +-
 .../pulsar/broker/admin/AdminResource.java    |   5 +
 .../broker/admin/impl/NamespacesBase.java     | 156 ++++++++----------
 .../broker/admin/impl/PackagesBase.java       |  11 --
 .../admin/impl/PersistentTopicsBase.java      |  48 +-----
 .../pulsar/broker/service/BrokerService.java  |   2 +-
 .../AuthorizationProducerConsumerTest.java    |   9 +-
 7 files changed, 93 insertions(+), 147 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
index 203f7fe427745..c96e683d8e81c 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
@@ -261,9 +261,12 @@ public CompletableFuture revokePermissionAsync(TopicName topicName, String
             return pulsarResources.getNamespaceResources()
                     .setPoliciesAsync(topicName.getNamespaceObject(), policies -> {
                         policies.auth_policies.getTopicAuthentication()
-                                .computeIfPresent(topicName.toString(), (k, v) -> {
-                                        v.remove(role);
-                                        return null;
+                                .computeIfPresent(topicName.toString(), (topicNameUri, roles) -> {
+                                        roles.remove(role);
+                                        if (roles.isEmpty()) {
+                                            return  null;
+                                        }
+                                        return roles;
                                 });
                         return policies;
                     }).whenComplete((__, ex) -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
index 4190b4c486a4c..828b18c2df9ab 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
@@ -42,6 +42,7 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.authorization.AuthorizationService;
 import org.apache.pulsar.broker.service.plugin.InvalidEntryFilterException;
 import org.apache.pulsar.broker.web.PulsarWebResource;
 import org.apache.pulsar.broker.web.RestException;
@@ -847,4 +848,8 @@ protected List filterSystemTopic(List topics, boolean includeSys
                 .filter(topic -> includeSystemTopic ? true : !pulsar().getBrokerService().isSystemTopic(topic))
                 .collect(Collectors.toList());
     }
+
+    protected AuthorizationService getAuthorizationService() {
+        return pulsar().getBrokerService().getAuthorizationService();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 97029eb5ce128..29c612880c153 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -56,7 +56,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.admin.AdminResource;
-import org.apache.pulsar.broker.authorization.AuthorizationService;
 import org.apache.pulsar.broker.loadbalance.LeaderBroker;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.service.BrokerServiceException;
@@ -570,109 +569,86 @@ protected CompletableFuture internalDeleteNamespaceBundleAsync(String bund
     }
 
     protected CompletableFuture internalGrantPermissionOnNamespaceAsync(String role, Set actions) {
-        AuthorizationService authService = pulsar().getBrokerService().getAuthorizationService();
-        if (null != authService) {
-            return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION)
-                    .thenAccept(__ -> {
-                        checkNotNull(role, "Role should not be null");
-                        checkNotNull(actions, "Actions should not be null");
-                    }).thenCompose(__ ->
-                            authService.grantPermissionAsync(namespaceName, actions, role, null))
-                    .thenAccept(unused ->
-                            log.info("[{}] Successfully granted access for role {}: {} - namespaceName {}",
-                                    clientAppId(), role, actions, namespaceName))
-                    .exceptionally(ex -> {
-                        Throwable realCause = FutureUtil.unwrapCompletionException(ex);
-                        //The IllegalArgumentException and the IllegalStateException were historically thrown by the
-                        // grantPermissionAsync method, so we catch them here to ensure backwards compatibility.
-                        if (realCause instanceof MetadataStoreException.NotFoundException
-                                || realCause instanceof IllegalArgumentException) {
-                            log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(),
-                                    namespaceName, ex);
-                            throw new RestException(Status.NOT_FOUND, "Topic's namespace does not exist");
-                        } else if (realCause instanceof MetadataStoreException.BadVersionException
-                                || realCause instanceof IllegalStateException) {
-                            log.warn("[{}] Failed to set permissions for namespace {}: {}",
-                                    clientAppId(), namespaceName, ex.getCause().getMessage(), ex);
-                            throw new RestException(Status.CONFLICT, "Concurrent modification");
-                        } else {
-                            log.error("[{}] Failed to get permissions for namespace {}",
-                                    clientAppId(), namespaceName, ex);
-                            throw new RestException(realCause);
-                        }
-                    });
-        } else {
-            String msg = "Authorization is not enabled";
-            return FutureUtil.failedFuture(new RestException(Status.NOT_IMPLEMENTED, msg));
-        }
+        return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION)
+                .thenAccept(__ -> {
+                    checkNotNull(role, "Role should not be null");
+                    checkNotNull(actions, "Actions should not be null");
+                }).thenCompose(__ ->
+                        getAuthorizationService().grantPermissionAsync(namespaceName, actions, role, null))
+                .thenAccept(unused ->
+                        log.info("[{}] Successfully granted access for role {}: {} - namespaceName {}",
+                                clientAppId(), role, actions, namespaceName))
+                .exceptionally(ex -> {
+                    Throwable realCause = FutureUtil.unwrapCompletionException(ex);
+                    //The IllegalArgumentException and the IllegalStateException were historically thrown by the
+                    // grantPermissionAsync method, so we catch them here to ensure backwards compatibility.
+                    if (realCause instanceof MetadataStoreException.NotFoundException
+                            || realCause instanceof IllegalArgumentException) {
+                        log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(),
+                                namespaceName, ex);
+                        throw new RestException(Status.NOT_FOUND, "Topic's namespace does not exist");
+                    } else if (realCause instanceof MetadataStoreException.BadVersionException
+                            || realCause instanceof IllegalStateException) {
+                        log.warn("[{}] Failed to set permissions for namespace {}: {}",
+                                clientAppId(), namespaceName, ex.getCause().getMessage(), ex);
+                        throw new RestException(Status.CONFLICT, "Concurrent modification");
+                    } else {
+                        log.error("[{}] Failed to get permissions for namespace {}",
+                                clientAppId(), namespaceName, ex);
+                        throw new RestException(realCause);
+                    }
+                });
     }
 
 
     protected CompletableFuture internalGrantPermissionOnSubscriptionAsync(String subscription,
                                                                                 Set roles) {
-        AuthorizationService authService = pulsar().getBrokerService().getAuthorizationService();
-        if (null != authService) {
-            return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION)
-                    .thenAccept(__ -> {
-                        checkNotNull(subscription, "Subscription should not be null");
-                        checkNotNull(roles, "Roles should not be null");
-                    })
-                    .thenCompose(__ -> authService.grantSubscriptionPermissionAsync(namespaceName, subscription,
-                            roles, null))
-                    .thenAccept(unused -> {
-                        log.info("[{}] Successfully granted permission on subscription for role {}:{} - "
-                                + "namespaceName {}", clientAppId(), roles, subscription, namespaceName);
-                    })
-                    .exceptionally(ex -> {
-                        Throwable realCause = FutureUtil.unwrapCompletionException(ex);
-                        //The IllegalArgumentException and the IllegalStateException were historically thrown by the
-                        // grantPermissionAsync method, so we catch them here to ensure backwards compatibility.
-                        if (realCause.getCause() instanceof IllegalArgumentException) {
-                            log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(),
-                                    namespaceName);
-                            throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
-                        } else if (realCause.getCause() instanceof IllegalStateException) {
-                            log.warn("[{}] Failed to set permissions for namespace {}: concurrent modification",
-                                    clientAppId(), namespaceName);
-                            throw new RestException(Status.CONFLICT, "Concurrent modification");
-                        } else {
-                            log.error("[{}] Failed to get permissions for namespace {}",
-                                    clientAppId(), namespaceName, realCause);
-                            throw new RestException(realCause);
-                        }
-                    });
-        } else {
-            String msg = "Authorization is not enabled";
-            return FutureUtil.failedFuture(new RestException(Status.NOT_IMPLEMENTED, msg));
-        }
+        return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION)
+                .thenAccept(__ -> {
+                    checkNotNull(subscription, "Subscription should not be null");
+                    checkNotNull(roles, "Roles should not be null");
+                })
+                .thenCompose(__ -> getAuthorizationService()
+                        .grantSubscriptionPermissionAsync(namespaceName, subscription, roles, null))
+                .thenAccept(unused -> {
+                    log.info("[{}] Successfully granted permission on subscription for role {}:{} - "
+                            + "namespaceName {}", clientAppId(), roles, subscription, namespaceName);
+                })
+                .exceptionally(ex -> {
+                    Throwable realCause = FutureUtil.unwrapCompletionException(ex);
+                    //The IllegalArgumentException and the IllegalStateException were historically thrown by the
+                    // grantPermissionAsync method, so we catch them here to ensure backwards compatibility.
+                    if (realCause.getCause() instanceof IllegalArgumentException) {
+                        log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(),
+                                namespaceName);
+                        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
+                    } else if (realCause.getCause() instanceof IllegalStateException) {
+                        log.warn("[{}] Failed to set permissions for namespace {}: concurrent modification",
+                                clientAppId(), namespaceName);
+                        throw new RestException(Status.CONFLICT, "Concurrent modification");
+                    } else {
+                        log.error("[{}] Failed to get permissions for namespace {}",
+                                clientAppId(), namespaceName, realCause);
+                        throw new RestException(realCause);
+                    }
+                });
     }
 
     protected CompletableFuture internalRevokePermissionsOnNamespaceAsync(String role) {
         return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.REVOKE_PERMISSION)
                 .thenAccept(__ -> checkNotNull(role, "Role should not be null"))
-                .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync())
-                .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> {
-                    policies.auth_policies.getNamespaceAuthentication().remove(role);
-                    return policies;
-                }));
+                .thenCompose(__ -> getAuthorizationService().revokePermissionAsync(namespaceName, role));
     }
 
     protected CompletableFuture internalRevokePermissionsOnSubscriptionAsync(String subscriptionName,
                                                                                   String role) {
-        AuthorizationService authService = pulsar().getBrokerService().getAuthorizationService();
-        if (null != authService) {
-            return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.REVOKE_PERMISSION)
-                    .thenAccept(__ -> {
-                        checkNotNull(subscriptionName, "SubscriptionName should not be null");
-                        checkNotNull(role, "Role should not be null");
-                    })
-                    .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync())
-                    .thenCompose(__ -> authService.revokeSubscriptionPermissionAsync(namespaceName,
-                            subscriptionName, role, null/* additional auth-data json */));
-        } else {
-            String msg = "Authorization is not enabled";
-            return FutureUtil.failedFuture(new RestException(Status.NOT_IMPLEMENTED, msg));
-        }
+        return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.REVOKE_PERMISSION)
+                .thenAccept(__ -> {
+                    checkNotNull(subscriptionName, "SubscriptionName should not be null");
+                    checkNotNull(role, "Role should not be null");
+                })
+                .thenCompose(__ -> getAuthorizationService().revokeSubscriptionPermissionAsync(namespaceName,
+                        subscriptionName, role, null/* additional auth-data json */));
     }
 
     protected CompletableFuture> internalGetNamespaceReplicationClustersAsync() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java
index c5b0f60e91c5e..c4f1db3205038 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java
@@ -27,7 +27,6 @@
 import javax.ws.rs.core.StreamingOutput;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.admin.AdminResource;
-import org.apache.pulsar.broker.authorization.AuthorizationService;
 import org.apache.pulsar.broker.web.RestException;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.policies.data.NamespaceOperation;
@@ -40,8 +39,6 @@
 @Slf4j
 public class PackagesBase extends AdminResource {
 
-    private AuthorizationService authorizationService;
-
     private PackagesManagement getPackagesManagement() {
         return pulsar().getPackagesManagement();
     }
@@ -197,12 +194,4 @@ private CompletableFuture checkPermissions(String tenant, String namespace
         }
         return future;
     }
-
-    private AuthorizationService getAuthorizationService() {
-        if (authorizationService == null) {
-            authorizationService = pulsar().getBrokerService().getAuthorizationService();
-            return authorizationService;
-        }
-        return authorizationService;
-    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index e0f168eb8d842..03421b820dafb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -319,45 +319,10 @@ protected void internalGrantPermissionsOnTopic(final AsyncResponse asyncResponse
                 });
     }
 
-    private CompletableFuture revokePermissionsAsync(String topicUri, String role, boolean force) {
-        return namespaceResources().getPoliciesAsync(namespaceName).thenCompose(
-                policiesOptional -> {
-                    Policies policies = policiesOptional.orElseThrow(() ->
-                            new RestException(Status.NOT_FOUND, "Namespace does not exist"));
-                    if (!policies.auth_policies.getTopicAuthentication().containsKey(topicUri)
-                            || !policies.auth_policies.getTopicAuthentication().get(topicUri).containsKey(role)) {
-                        log.warn("[{}] Failed to revoke permission from role {} on topic: Not set at topic level {}",
-                                clientAppId(), role, topicUri);
-                        if (force) {
-                            return CompletableFuture.completedFuture(null);
-                        } else {
-                            return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED,
-                                    "Permissions are not set at the topic level"));
-                        }
-                    }
-                    // Write the new policies to metadata store
-                    return namespaceResources().setPoliciesAsync(namespaceName, p -> {
-                        p.auth_policies.getTopicAuthentication().computeIfPresent(topicUri, (k, roles) -> {
-                            roles.remove(role);
-                            if (roles.isEmpty()) {
-                                return null;
-                            }
-                            return roles;
-                        });
-                        return p;
-                    }).thenAccept(__ ->
-                            log.info("[{}] Successfully revoke access for role {} - topic {}", clientAppId(), role,
-                            topicUri)
-                    );
-                }
-        );
-    }
-
     protected void internalRevokePermissionsOnTopic(AsyncResponse asyncResponse, String role) {
         // This operation should be reading from zookeeper and it should be allowed without having admin privileges
         validateAdminAccessForTenantAsync(namespaceName.getTenant())
-                .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync().thenCompose(unused1 ->
-            getPartitionedTopicMetadataAsync(topicName, true, false)
+                .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, true, false)
                 .thenCompose(metadata -> {
                     int numPartitions = metadata.partitions;
                     CompletableFuture future = CompletableFuture.completedFuture(null);
@@ -365,13 +330,14 @@ protected void internalRevokePermissionsOnTopic(AsyncResponse asyncResponse, Str
                         for (int i = 0; i < numPartitions; i++) {
                             TopicName topicNamePartition = topicName.getPartition(i);
                             future = future.thenComposeAsync(unused ->
-                                    revokePermissionsAsync(topicNamePartition.toString(), role, true));
+                                    getAuthorizationService().revokePermissionAsync(topicNamePartition, role));
                         }
                     }
-                    return future.thenComposeAsync(unused -> revokePermissionsAsync(topicName.toString(), role, false))
-                            .thenAccept(unused -> asyncResponse.resume(Response.noContent().build()));
-                }))
-            ).exceptionally(ex -> {
+                    return future.thenComposeAsync(unused ->
+                                    getAuthorizationService().revokePermissionAsync(topicName, role))
+                                 .thenAccept(unused -> asyncResponse.resume(Response.noContent().build()));
+                })
+                ).exceptionally(ex -> {
                     Throwable realCause = FutureUtil.unwrapCompletionException(ex);
                     log.error("[{}] Failed to revoke permissions for topic {}", clientAppId(), topicName, realCause);
                     resumeAsyncResponseExceptionally(asyncResponse, realCause);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 3dc7718c32a74..d952f7d65743a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -220,7 +220,7 @@ public class BrokerService implements Closeable {
 
     private final ConcurrentLinkedQueue pendingTopicLoadingQueue;
 
-    private AuthorizationService authorizationService = null;
+    private AuthorizationService authorizationService;
     private final ScheduledExecutorService statsUpdater;
     @Getter
     private final ScheduledExecutorService backlogQuotaChecker;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
index 01fa64b1bc62e..bdbe8efc8e6ef 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
@@ -696,22 +696,29 @@ public void testRevokePermission() throws Exception {
         TopicName topicName = TopicName.get("persistent://public/default/t1");
         NamespaceName namespaceName = NamespaceName.get("public/default");
         String role = "test-role";
+        String role2 = "test-role-2";
         Set actions = Sets.newHashSet(AuthAction.produce, AuthAction.consume);
         Assert.assertFalse(authorizationService.canProduce(topicName, role, null));
         Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1"));
         authorizationService.grantPermissionAsync(topicName, actions, role, "auth-json").get();
+        authorizationService.grantPermissionAsync(topicName, actions, role2, "auth-json").get();
         Assert.assertTrue(authorizationService.canProduce(topicName, role, null));
         Assert.assertTrue(authorizationService.canConsume(topicName, role, null, "sub1"));
 
         authorizationService.revokePermissionAsync(topicName, role).get();
         Assert.assertFalse(authorizationService.canProduce(topicName, role, null));
         Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1"));
+        Assert.assertTrue(authorizationService.canProduce(topicName, role2, null));
+        Assert.assertTrue(authorizationService.canConsume(topicName, role2, null, "sub1"));
+
+        authorizationService.revokePermissionAsync(topicName, role2).get();
+        Assert.assertFalse(authorizationService.canProduce(topicName, role2, null));
+        Assert.assertFalse(authorizationService.canConsume(topicName, role2, null, "sub1"));
 
         authorizationService.grantPermissionAsync(namespaceName, actions, role, null).get();
         Assert.assertTrue(authorizationService.allowNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPIC, role, null).get());
         authorizationService.revokePermissionAsync(namespaceName, role).get();
         Assert.assertFalse(authorizationService.allowNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPIC, role, null).get());
-
         log.info("-- Exiting {} test --", methodName);
     }
 

From 12e6dd558b6ca1a22c228c8567f466d3d6aef024 Mon Sep 17 00:00:00 2001
From: Penghui Li 
Date: Tue, 6 Jun 2023 11:08:36 +0800
Subject: [PATCH 377/404] [fix][misc] Use ubuntu 22.04 for Pulsar images
 (#20475)

### Motivation

Upgrade Ubuntu to 22.04 to fix some CVEs

- CVE-2017-14063: https://nvd.nist.gov/vuln/detail/cve-2017-14063
- CVE-2021-27291: https://nvd.nist.gov/vuln/detail/CVE-2021-27291
- CVE-2021-20270: https://nvd.nist.gov/vuln/detail/CVE-2021-20270
---
 build/docker/Dockerfile                        | 2 +-
 docker/pulsar/Dockerfile                       | 2 +-
 pom.xml                                        | 2 +-
 tests/docker-images/java-test-image/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index 89b8f25cd06b2..674167b326378 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # prepare the directory for pulsar related files
 RUN mkdir /pulsar
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index 01e53e0152ac6..a5b294063d376 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -49,7 +49,7 @@ RUN chmod g+w /pulsar/trino
 ### Create 2nd stage from Ubuntu image
 ### and add OpenJDK and Python dependencies (for Pulsar functions)
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt
diff --git a/pom.xml b/pom.xml
index 6f1d4d7118608..6cd0a4fc5b9b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,7 +82,7 @@ flexible messaging model and an intuitive client API.
     ${maven.compiler.target}
     8
 
-    2.10.1
+    3.2.0
 
     
     **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java
diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile
index 9e852050edf2e..5821e6eeaeae7 100644
--- a/tests/docker-images/java-test-image/Dockerfile
+++ b/tests/docker-images/java-test-image/Dockerfile
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 RUN groupadd -g 10001 pulsar
 RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar

From da04f24b6e81f05ace04fb4ae0d9a720b44c845f Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 6 Jun 2023 00:03:05 -0500
Subject: [PATCH 378/404] [fix][test] Replace calls to Auth0 with calls to
 wiremock (#20500)

---
 .../broker/auth/MockOIDCIdentityProvider.java | 150 ++++++++++++++++++
 ...uth2AuthenticatedProducerConsumerTest.java | 131 +++------------
 .../token/credentials_file.json               |   4 +-
 pulsar-proxy/pom.xml                          |   6 +
 .../proxy/server/ProxyTlsTestWithAuth.java    |  15 +-
 pulsar-testclient/pom.xml                     |   7 +
 .../Oauth2PerformanceTransactionTest.java     |  24 ++-
 .../token/credentials_file.json               |   4 +-
 8 files changed, 208 insertions(+), 133 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java
new file mode 100644
index 0000000000000..5d29c443d2bde
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.auth;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.matching;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.common.FileSource;
+import com.github.tomakehurst.wiremock.extension.Parameters;
+import com.github.tomakehurst.wiremock.extension.ResponseTransformer;
+import com.github.tomakehurst.wiremock.http.Request;
+import com.github.tomakehurst.wiremock.http.Response;
+import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.impl.DefaultJwtBuilder;
+import io.jsonwebtoken.security.Keys;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.util.Base64;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Mock OIDC (and therefore OAuth2) server for testing. Note that the client_id is mapped to the token's subject claim.
+ */
+public class MockOIDCIdentityProvider {
+    private final WireMockServer server;
+    private final PublicKey publicKey;
+    private final String audience;
+    public MockOIDCIdentityProvider(String clientSecret, String audience, long tokenTTLMillis) {
+        this.audience = audience;
+        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
+        publicKey = keyPair.getPublic();
+        server = new WireMockServer(wireMockConfig().port(0)
+                .extensions(new OAuth2Transformer(keyPair, tokenTTLMillis)));
+        server.start();
+
+        // Set up a correct openid-configuration that points to the next stub
+        server.stubFor(
+                get(urlEqualTo("/.well-known/openid-configuration"))
+                        .willReturn(aResponse()
+                                .withHeader("Content-Type", "application/json")
+                                .withBody("""
+                                        {
+                                          "issuer": "%s",
+                                          "token_endpoint": "%s/oauth/token"
+                                        }
+                                        """.replace("%s", server.baseUrl()))));
+
+        // Only respond when the client sends the expected request body
+        server.stubFor(post(urlEqualTo("/oauth/token"))
+                .withRequestBody(matching(".*grant_type=client_credentials.*"))
+                .withRequestBody(matching(".*audience=" + URLEncoder.encode(audience, StandardCharsets.UTF_8) + ".*"))
+                .withRequestBody(matching(".*client_id=.*"))
+                .withRequestBody(matching(".*client_secret=" + clientSecret + "(&.*|$)"))
+                .willReturn(aResponse().withTransformers("o-auth-token-transformer").withStatus(200)));
+    }
+
+    public void stop() {
+        server.stop();
+    }
+
+    public String getBase64EncodedPublicKey() {
+        return Base64.getEncoder().encodeToString(publicKey.getEncoded());
+    }
+
+    public String getIssuer() {
+        return server.baseUrl();
+    }
+
+    class OAuth2Transformer extends ResponseTransformer {
+
+        private final PrivateKey privateKey;
+        private final long tokenTTL;
+
+        private final Pattern clientIdToRolePattern = Pattern.compile("client_id=([A-Za-z0-9-]*)(&|$)");
+
+        OAuth2Transformer(KeyPair key, long tokenTTLMillis) {
+            this.privateKey = key.getPrivate();
+            this.tokenTTL = tokenTTLMillis;
+        }
+
+        @Override
+        public Response transform(Request request, Response response, FileSource files, Parameters parameters) {
+            Matcher m = clientIdToRolePattern.matcher(request.getBodyAsString());
+            if (m.find()) {
+                String role = m.group(1);
+                return Response.Builder.like(response).but().body("""
+                        {
+                          "access_token": "%s",
+                          "expires_in": %d,
+                          "token_type":"Bearer"
+                        }
+                        """.formatted(generateToken(role),
+                        TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build();
+            } else {
+                return Response.Builder.like(response).but().body("Invalid request").status(400).build();
+            }
+        }
+
+        @Override
+        public String getName() {
+            return "o-auth-token-transformer";
+        }
+
+        @Override
+        public boolean applyGlobally() {
+            return false;
+        }
+
+        private String generateToken(String role) {
+            long now = System.currentTimeMillis();
+            DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
+            defaultJwtBuilder.setHeaderParam("typ", "JWT");
+            defaultJwtBuilder.setHeaderParam("alg", "RS256");
+            defaultJwtBuilder.setIssuer(server.baseUrl());
+            defaultJwtBuilder.setSubject(role);
+            defaultJwtBuilder.setAudience(audience);
+            defaultJwtBuilder.setIssuedAt(new Date(now));
+            defaultJwtBuilder.setNotBefore(new Date(now));
+            defaultJwtBuilder.setExpiration(new Date(now + tokenTTL));
+            defaultJwtBuilder.signWith(privateKey);
+            return defaultJwtBuilder.compact();
+        }
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java
index cf85ddd913b8b..fdf41c4a6ada1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java
@@ -18,38 +18,20 @@
  */
 package org.apache.pulsar.client.api;
 
-import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
-import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.post;
-import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
-import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
 import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
-import com.github.tomakehurst.wiremock.WireMockServer;
-import com.github.tomakehurst.wiremock.common.FileSource;
-import com.github.tomakehurst.wiremock.extension.Parameters;
-import com.github.tomakehurst.wiremock.extension.ResponseTransformer;
-import com.github.tomakehurst.wiremock.http.Request;
-import com.github.tomakehurst.wiremock.http.Response;
 import com.google.common.collect.Sets;
-import io.jsonwebtoken.SignatureAlgorithm;
-import io.jsonwebtoken.impl.DefaultJwtBuilder;
-import io.jsonwebtoken.security.Keys;
 import java.net.URI;
 import java.net.URL;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.security.KeyPair;
-import java.security.PrivateKey;
 import java.time.Duration;
-import java.util.Base64;
-import java.util.Date;
 import java.util.HashSet;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.impl.ProducerImpl;
@@ -60,7 +42,9 @@
 import org.awaitility.Awaitility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -74,54 +58,28 @@
 public class TokenOauth2AuthenticatedProducerConsumerTest extends ProducerConsumerBase {
     private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class);
 
-    private WireMockServer server;
-
-    private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients";
+    private MockOIDCIdentityProvider server;
 
     // Credentials File, which contains "client_id" and "client_secret"
     private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json";
-    private final String AUDIENCE = "https://dev-kt-aa9ne.us.auth0.com/api/v2/";
+    private final String audience = "my-pulsar-cluster";
+
+    @BeforeClass(alwaysRun = true)
+    protected void setupClass() {
+        String clientSecret = "super-secret-client-secret";
+        server = new MockOIDCIdentityProvider(clientSecret, audience, 3000);
+    }
 
     @BeforeMethod(alwaysRun = true)
     @Override
     protected void setup() throws Exception {
-        // Create the token key pair
-        KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256);
-
-        // Start mocked OAuth2 server
-        server = new WireMockServer(wireMockConfig().port(0).extensions(new OAuth2Transformer(keyPair, 3000)));
-        server.start();
-
-        // Set up a correct openid-configuration that points to the next stub
-        server.stubFor(
-                get(urlEqualTo("/.well-known/openid-configuration"))
-                        .willReturn(aResponse()
-                                .withHeader("Content-Type", "application/json")
-                                .withBody("""
-                                        {
-                                          "issuer": "%s",
-                                          "token_endpoint": "%s/oauth/token"
-                                        }
-                                        """.replace("%s", server.baseUrl()))));
-
-        // Only respond when the client sends the expected request body
-        server.stubFor(
-                post(urlEqualTo("/oauth/token"))
-                        .withRequestBody(
-                                equalTo("audience=https%3A%2F%2Fdev-kt-aa9ne.us.auth0.com%2Fapi%2Fv2%2F&"
-                                        + "client_id=Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x&"
-                                        + "client_secret=rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N"
-                                        + "_poGAb&grant_type=client_credentials"))
-                        .willReturn(aResponse()
-                                .withTransformers("o-auth-token-transformer")
-                                .withStatus(200)));
-
         conf.setAuthenticationEnabled(true);
         conf.setAuthorizationEnabled(true);
         conf.setAuthenticationRefreshCheckSeconds(1);
 
         Set superUserRoles = new HashSet<>();
-        superUserRoles.add(ADMIN_ROLE);
+        // Matches the role in th credentials file
+        superUserRoles.add("my-admin-role");
         conf.setSuperUserRoles(superUserRoles);
 
         Set providers = new HashSet<>();
@@ -131,16 +89,15 @@ protected void setup() throws Exception {
         conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName());
         conf.setBrokerClientAuthenticationParameters("{\n"
                 + "  \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n"
-                + "  \"issuerUrl\": \"" + server.baseUrl() + "\",\n"
-                + "  \"audience\": \"" + AUDIENCE + "\",\n"
+                + "  \"issuerUrl\": \"" + server.getIssuer() + "\",\n"
+                + "  \"audience\": \"" + audience + "\",\n"
                 + "}\n");
 
         conf.setClusterName("test");
 
         // Set provider domain name
         Properties properties = new Properties();
-        properties.setProperty("tokenPublicKey", "data:;base64,"
-                + Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded()));
+        properties.setProperty("tokenPublicKey", "data:;base64," + server.getBase64EncodedPublicKey());
 
         conf.setProperties(properties);
         super.init();
@@ -153,9 +110,9 @@ protected final void clientSetup() throws Exception {
 
         // AuthenticationOAuth2
         Authentication authentication = AuthenticationFactoryOAuth2.clientCredentials(
-                new URL(server.baseUrl()),
+                new URL(server.getIssuer()),
                 path.toUri().toURL(),  // key file path
-                AUDIENCE
+                audience
         );
 
         admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString())
@@ -171,6 +128,10 @@ protected final void clientSetup() throws Exception {
     @Override
     protected void cleanup() throws Exception {
         super.internalCleanup();
+    }
+
+    @AfterClass(alwaysRun = true)
+    protected void cleanupAfterClass() {
         server.stop();
     }
 
@@ -292,52 +253,4 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception {
         consumer.acknowledgeCumulative(msg);
         consumer.close();
     }
-
-    class OAuth2Transformer extends ResponseTransformer {
-
-        private final PrivateKey privateKey;
-        private final long tokenTTL;
-
-        OAuth2Transformer(KeyPair key, long tokenTTLMillis) {
-            this.privateKey = key.getPrivate();
-            this.tokenTTL = tokenTTLMillis;
-        }
-
-        @Override
-        public Response transform(Request request, Response response, FileSource files, Parameters parameters) {
-            return Response.Builder.like(response).but().body("""
-                                                              {
-                                                                "access_token": "%s",
-                                                                "expires_in": %d,
-                                                                "token_type":"Bearer"
-                                                              }
-                                                              """.formatted(generateToken(),
-                    TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build();
-        }
-
-        @Override
-        public String getName() {
-            return "o-auth-token-transformer";
-        }
-
-        @Override
-        public boolean applyGlobally() {
-            return false;
-        }
-
-        private String generateToken() {
-            long now = System.currentTimeMillis();
-            DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder();
-            defaultJwtBuilder.setHeaderParam("typ", "JWT");
-            defaultJwtBuilder.setHeaderParam("alg", "RS256");
-            defaultJwtBuilder.setIssuer(server.baseUrl());
-            defaultJwtBuilder.setSubject(ADMIN_ROLE);
-            defaultJwtBuilder.setAudience(AUDIENCE);
-            defaultJwtBuilder.setIssuedAt(new Date(now));
-            defaultJwtBuilder.setNotBefore(new Date(now));
-            defaultJwtBuilder.setExpiration(new Date(now + tokenTTL));
-            defaultJwtBuilder.signWith(privateKey);
-            return defaultJwtBuilder.compact();
-        }
-    }
 }
diff --git a/pulsar-broker/src/test/resources/authentication/token/credentials_file.json b/pulsar-broker/src/test/resources/authentication/token/credentials_file.json
index db1eccd8eb678..d12e786b7cdb5 100644
--- a/pulsar-broker/src/test/resources/authentication/token/credentials_file.json
+++ b/pulsar-broker/src/test/resources/authentication/token/credentials_file.json
@@ -1,4 +1,4 @@
 {
-  "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x",
-  "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb"
+  "client_id":"my-admin-role",
+  "client_secret":"super-secret-client-secret"
 }
diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml
index 286c0ac0912a0..a607b9e835f8e 100644
--- a/pulsar-proxy/pom.xml
+++ b/pulsar-proxy/pom.xml
@@ -185,6 +185,12 @@
       testcontainers
       test
     
+    
+      com.github.tomakehurst
+      wiremock-jre8
+      ${wiremock.version}
+      test
+    
   
   
     
diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
index f77c0eeb2d41c..0f1fa74a20916 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java
@@ -24,6 +24,7 @@
 import java.io.FileWriter;
 import java.util.Optional;
 
+import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider;
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.broker.authentication.AuthenticationService;
 import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
@@ -38,17 +39,21 @@ public class ProxyTlsTestWithAuth extends MockedPulsarServiceBaseTest {
     private ProxyService proxyService;
     private ProxyConfiguration proxyConfig = new ProxyConfiguration();
 
+    private MockOIDCIdentityProvider server;
+
     @Override
     @BeforeClass
     protected void setup() throws Exception {
         internalSetup();
+        String clientSecret = "super-secret-client-secret";
+        server = new MockOIDCIdentityProvider(clientSecret, "an-audience", 3000);
 
         File tempFile = File.createTempFile("oauth2", ".tmp");
         tempFile.deleteOnExit();
         FileWriter writer = new FileWriter(tempFile);
         writer.write("{\n" +
-            "  \"client_id\":\"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x\",\n" +
-            "  \"client_secret\":\"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb\"\n" +
+            "  \"client_id\":\"my-user\",\n" +
+            "  \"client_secret\":\"" + clientSecret + "\"\n" +
             "}");
         writer.flush();
         writer.close();
@@ -65,8 +70,8 @@ protected void setup() throws Exception {
         proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE);
         proxyConfig.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2");
         proxyConfig.setBrokerClientAuthenticationParameters("{\"grant_type\":\"client_credentials\"," +
-            " \"issuerUrl\":\"https://dev-kt-aa9ne.us.auth0.com\"," +
-            " \"audience\": \"https://dev-kt-aa9ne.us.auth0.com/api/v2/\"," +
+            " \"issuerUrl\":\"" + server.getIssuer() + "\"," +
+            " \"audience\": \"an-audience\"," +
             " \"privateKey\":\"file://" + tempFile.getAbsolutePath() + "\"}");
 
         proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService(
@@ -81,8 +86,8 @@ protected void setup() throws Exception {
     @AfterClass(alwaysRun = true)
     protected void cleanup() throws Exception {
         internalCleanup();
-
         proxyService.close();
+        server.stop();
     }
 
     @Test
diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml
index 3848586605301..ce03dd444e101 100644
--- a/pulsar-testclient/pom.xml
+++ b/pulsar-testclient/pom.xml
@@ -112,6 +112,13 @@
 			test
 		
 
+		
+			com.github.tomakehurst
+			wiremock-jre8
+			${wiremock.version}
+			test
+		
+
 
 	
 
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java
index 05c9b069aca87..d19c4b3d104b7 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java
@@ -32,6 +32,7 @@
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.api.Consumer;
@@ -42,7 +43,6 @@
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.TokenOauth2AuthenticatedProducerConsumerTest;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
@@ -61,32 +61,25 @@ public class Oauth2PerformanceTransactionTest extends ProducerConsumerBase {
     private final String testNamespace = "perf";
     private final String myNamespace = testTenant + "/" + testNamespace;
     private final String testTopic = "persistent://" + myNamespace + "/test-";
-    private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class);
-
-    // public key in oauth2 server to verify the client passed in token. get from https://jwt.io/
-    private final String TOKEN_TEST_PUBLIC_KEY = "data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/"
-            + "4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRm"
-            + "WjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6"
-            + "NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDH"
-            + "y5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB";
-
-    private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients";
+    private static final Logger log = LoggerFactory.getLogger(Oauth2PerformanceTransactionTest.class);
 
     // Credentials File, which contains "client_id" and "client_secret"
     private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json";
 
     private final String authenticationPlugin = "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2";
 
+    private MockOIDCIdentityProvider server;
     private String authenticationParameters;
 
     @BeforeMethod(alwaysRun = true)
     @Override
     protected void setup() throws Exception {
+        server = new MockOIDCIdentityProvider("a-client-secret", "my-test-audience", 30000);
         Path path = Paths.get(CREDENTIALS_FILE).toAbsolutePath();
         HashMap params = new HashMap<>();
-        params.put("issuerUrl", new URL("https://dev-kt-aa9ne.us.auth0.com"));
+        params.put("issuerUrl", server.getIssuer());
         params.put("privateKey", path.toUri().toURL());
-        params.put("audience", "https://dev-kt-aa9ne.us.auth0.com/api/v2/");
+        params.put("audience", "my-test-audience");
         ObjectMapper jsonMapper = ObjectMapperFactory.create();
         authenticationParameters = jsonMapper.writeValueAsString(params);
 
@@ -96,7 +89,7 @@ protected void setup() throws Exception {
         conf.setAuthenticationRefreshCheckSeconds(5);
 
         Set superUserRoles = new HashSet<>();
-        superUserRoles.add(ADMIN_ROLE);
+        superUserRoles.add("superuser");
         conf.setSuperUserRoles(superUserRoles);
 
         Set providers = new HashSet<>();
@@ -107,7 +100,7 @@ protected void setup() throws Exception {
 
         // Set provider domain name
         Properties properties = new Properties();
-        properties.setProperty("tokenPublicKey", TOKEN_TEST_PUBLIC_KEY);
+        properties.setProperty("tokenPublicKey", server.getBase64EncodedPublicKey());
 
         conf.setProperties(properties);
 
@@ -127,6 +120,7 @@ protected void setup() throws Exception {
     @Override
     protected void cleanup() throws Exception {
         super.internalCleanup();
+        server.stop();
     }
 
     // setup both admin and pulsar client
diff --git a/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json b/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json
index 698ad9d93e3da..92ab1c3b7cd87 100644
--- a/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json
+++ b/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json
@@ -1,5 +1,5 @@
 {
   "type": "client_credentials",
-  "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x",
-  "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb"
+  "client_id":"superuser",
+  "client_secret":"a-client-secret"
 }

From fff5e39a73eb07d8bdd73df5499bc134b277be11 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 6 Jun 2023 08:50:47 +0300
Subject: [PATCH 379/404] [improve][broker] Improve efficiency of checking
 message deletion (#20490)

---
 .../mledger/impl/ManagedCursorImpl.java          | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 1ce0403a54762..f30f9553e15e0 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -1474,9 +1474,8 @@ public Set asyncReplayEntries(Set positi
         lock.readLock().lock();
         try {
             positions.stream()
-                    .filter(position -> individualDeletedMessages.contains(((PositionImpl) position).getLedgerId(),
-                            ((PositionImpl) position).getEntryId())
-                            || ((PositionImpl) position).compareTo(markDeletePosition) <= 0)
+                    .filter(position -> ((PositionImpl) position).compareTo(markDeletePosition) <= 0
+                            || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId()))
                     .forEach(alreadyAcknowledgedPositions::add);
         } finally {
             lock.readLock().unlock();
@@ -2234,8 +2233,8 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
                     return;
                 }
 
-                if (individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId())
-                    || position.compareTo(markDeletePosition) <= 0) {
+                if (position.compareTo(markDeletePosition) <= 0
+                        || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId())) {
                     if (config.isDeletionAtBatchIndexLevelEnabled()) {
                         BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
                         if (bitSetRecyclable != null) {
@@ -3343,9 +3342,8 @@ public LongPairRangeSet getIndividuallyDeletedMessagesSet() {
 
     public boolean isMessageDeleted(Position position) {
         checkArgument(position instanceof PositionImpl);
-        return individualDeletedMessages.contains(((PositionImpl) position).getLedgerId(),
-                ((PositionImpl) position).getEntryId())
-                || ((PositionImpl) position).compareTo(markDeletePosition) <= 0;
+        return ((PositionImpl) position).compareTo(markDeletePosition) <= 0
+                || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
     }
 
     //this method will return a copy of the position's ack set
@@ -3437,7 +3435,7 @@ public Range getLastIndividualDeletedRange() {
     @Override
     public void trimDeletedEntries(List entries) {
         entries.removeIf(entry -> {
-            boolean isDeleted = ((PositionImpl) entry.getPosition()).compareTo(markDeletePosition) <= 0
+            boolean isDeleted = markDeletePosition.compareTo(entry.getLedgerId(), entry.getEntryId()) >= 0
                     || individualDeletedMessages.contains(entry.getLedgerId(), entry.getEntryId());
             if (isDeleted) {
                 entry.release();

From ad352a90f13173324feac816ee97ffff5140cf8f Mon Sep 17 00:00:00 2001
From: Rajan Dhabalia 
Date: Tue, 6 Jun 2023 03:58:40 -0700
Subject: [PATCH 380/404] [fix][cli] Fix logging noise while admin tool exit
 (#19884)

Co-authored-by: tison 
---
 .../java/org/apache/pulsar/admin/cli/PulsarAdminTool.java    | 2 +-
 .../java/org/apache/pulsar/common/util/ShutdownUtil.java     | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
index c06016be43883..e25520dceb9f8 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
@@ -306,7 +306,7 @@ private static void exit(int code) {
         if (allowSystemExit) {
             // we are using halt and not System.exit, we do not mind about shutdown hooks
             // they are only slowing down the tool
-            ShutdownUtil.triggerImmediateForcefulShutdown(code);
+            ShutdownUtil.triggerImmediateForcefulShutdown(code, false);
         } else {
             System.out.println("Exit code is " + code + " (System.exit not called, as we are in test mode)");
         }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java
index b461cfab0d1bc..ff12484707123 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java
@@ -47,8 +47,11 @@ public class ShutdownUtil {
      * @see Runtime#halt(int)
      */
     public static void triggerImmediateForcefulShutdown(int status) {
+        triggerImmediateForcefulShutdown(status, true);
+    }
+    public static void triggerImmediateForcefulShutdown(int status, boolean logging) {
         try {
-            if (status != 0) {
+            if (status != 0 && logging) {
                 log.warn("Triggering immediate shutdown of current process with status {}", status,
                         new Exception("Stacktrace for immediate shutdown"));
             }

From d7f355881b2b1eebf2be6ea262c202660d684fb7 Mon Sep 17 00:00:00 2001
From: Zike Yang 
Date: Tue, 6 Jun 2023 19:45:58 +0800
Subject: [PATCH 381/404] [fix][build] Fix the pulsar-all image may use the
 wrong upstream image (#20435)

Signed-off-by: Zike Yang 
Co-authored-by: Lari Hotari 
---
 docker/build.sh              |  2 +-
 docker/get-version.sh        |  2 +-
 docker/pom.xml               | 14 ++++++++++++++
 docker/publish.sh            | 24 ++++++++++++++++--------
 docker/pulsar-all/Dockerfile |  3 ++-
 docker/pulsar-all/pom.xml    | 14 +++++++++++++-
 docker/pulsar/pom.xml        | 17 ++++++++++++++---
 pom.xml                      | 27 ++++++++++++++++++++++++++-
 pulsar-common/pom.xml        |  3 ++-
 9 files changed, 89 insertions(+), 17 deletions(-)

diff --git a/docker/build.sh b/docker/build.sh
index d8ab4bea882c4..88be44f23e73f 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -18,7 +18,7 @@
 # under the License.
 #
 
-ROOT_DIR=$(git rev-parse --show-toplevel)
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )"
 cd $ROOT_DIR/docker
 
 mvn package -Pdocker,-main
diff --git a/docker/get-version.sh b/docker/get-version.sh
index 07145e7cf0c18..0b736baf3b270 100755
--- a/docker/get-version.sh
+++ b/docker/get-version.sh
@@ -18,7 +18,7 @@
 # under the License.
 #
 
-ROOT_DIR=$(git rev-parse --show-toplevel)
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )"
 
 pushd $ROOT_DIR > /dev/null
 
diff --git a/docker/pom.xml b/docker/pom.xml
index 882240925ef24..afe55f0fe57f0 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -60,6 +60,20 @@
         pulsar
         pulsar-all
       
+      
+        
+          
+            pl.project13.maven
+            git-commit-id-plugin
+            
+              false
+              true
+              true
+              false
+            
+          
+        
+      
     
   
 
diff --git a/docker/publish.sh b/docker/publish.sh
index 45b338d85f8ef..651fefc1498e9 100755
--- a/docker/publish.sh
+++ b/docker/publish.sh
@@ -18,7 +18,7 @@
 # under the License.
 #
 
-ROOT_DIR=$(git rev-parse --show-toplevel)
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )"
 cd $ROOT_DIR/docker
 
 # We should only publish images that are made from official and approved releases
@@ -49,6 +49,9 @@ fi
 
 MVN_VERSION=`./get-version.sh`
 echo "Pulsar version: ${MVN_VERSION}"
+GIT_COMMIT_ID_ABBREV=$(git rev-parse --short=7 HEAD 2>/dev/null || echo no-git)
+GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo no-git)
+IMAGE_TAG="${MVN_VERSION}-${GIT_COMMIT_ID_ABBREV}"
 
 if [[ -z ${DOCKER_REGISTRY} ]]; then
     docker_registry_org=${DOCKER_ORG}
@@ -62,16 +65,21 @@ set -x
 # Fail if any of the subsequent commands fail
 set -e
 
-docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:latest
-docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:latest
+if [[ "$GIT_BRANCH" == "master" ]]; then
+  docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:latest
+  docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:latest
+fi
 
-docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION
-docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION
+docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:$MVN_VERSION
+docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:$MVN_VERSION
 
 # Push all images and tags
-docker push ${docker_registry_org}/pulsar:latest
-docker push ${docker_registry_org}/pulsar-all:latest
+if [[ "$GIT_BRANCH" == "master" ]]; then
+  docker push ${docker_registry_org}/pulsar:latest
+  docker push ${docker_registry_org}/pulsar-all:latest
+fi
+
 docker push ${docker_registry_org}/pulsar:$MVN_VERSION
 docker push ${docker_registry_org}/pulsar-all:$MVN_VERSION
 
-echo "Finished pushing images to ${docker_registry_org}"
+echo "Finished pushing images to ${docker_registry_org}"
\ No newline at end of file
diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile
index 42431fc94a067..81ad74b65000f 100644
--- a/docker/pulsar-all/Dockerfile
+++ b/docker/pulsar-all/Dockerfile
@@ -17,6 +17,7 @@
 # under the License.
 #
 
+ARG PULSAR_IMAGE
 FROM busybox as pulsar-all
 
 ARG PULSAR_IO_DIR
@@ -26,6 +27,6 @@ ADD ${PULSAR_IO_DIR} /connectors
 ADD ${PULSAR_OFFLOADER_TARBALL} /
 RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders
 
-FROM apachepulsar/pulsar:latest
+FROM $PULSAR_IMAGE
 COPY --from=pulsar-all /connectors /pulsar/connectors
 COPY --from=pulsar-all /offloaders /pulsar/offloaders
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index 7a2f492632135..e616ac132d319 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -66,6 +66,17 @@
   
 
   
+    
+      git-commit-id-no-git
+      
+        
+          ${basedir}/../../.git/index
+        
+      
+      
+        no-git
+      
+    
     
       docker
       
@@ -144,11 +155,12 @@
                         ${project.basedir}
                         
                           latest
-                          ${project.version}
+                          ${project.version}-${git.commit.id.abbrev}
                         
                         
                           target/apache-pulsar-io-connectors-${project.version}-bin
                           target/pulsar-offloader-distribution-${project.version}-bin.tar.gz
+                          ${docker.organization}/pulsar:${project.version}-${git.commit.id.abbrev}
                         
                         
                           
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index e1c1503a3f381..85d86cff12523 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -48,11 +48,22 @@
   
 
   
-      mirror://mirrors.ubuntu.com/mirrors.txt
-      http://security.ubuntu.com/ubuntu/
+    mirror://mirrors.ubuntu.com/mirrors.txt
+    http://security.ubuntu.com/ubuntu/
   
 
   
+    
+      git-commit-id-no-git
+      
+        
+          ${basedir}/../../.git/index
+        
+      
+      
+        no-git
+      
+    
     
       docker
       
@@ -82,7 +93,7 @@
                         ${project.basedir}
                         
                           latest
-                          ${project.version}
+                          ${project.version}-${git.commit.id.abbrev}
                         
                         
                           
diff --git a/pom.xml b/pom.xml
index 6cd0a4fc5b9b3..7515aaf6863d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -286,7 +286,7 @@ flexible messaging model and an intuitive client API.
     1.1.0
     1.5.0
     3.1.2
-    4.0.2
+    4.9.10
     3.5.3
     1.7.0
     0.8.8
@@ -1571,6 +1571,31 @@ flexible messaging model and an intuitive client API.
         
       
 
+      
+        pl.project13.maven
+        git-commit-id-plugin
+        ${git-commit-id-plugin.version}
+        
+          
+            git-info
+            
+              revision
+            
+          
+        
+        
+          true
+          true
+          git
+          false
+          false
+          
+            false
+            false
+          
+        
+      
+
       
         com.mycila
         license-maven-plugin
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index a7d4dcf6beeca..cfc9cba7ede7f 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -194,7 +194,7 @@
       provided
       true
     
-    
+
     
       com.google.protobuf
 	  protobuf-java
@@ -296,6 +296,7 @@
           
         
         
+          false
           true
           git
           false

From f4386c868b3100487ee075c75f4cb78ff9c1d971 Mon Sep 17 00:00:00 2001
From: Ruguo Yu 
Date: Tue, 6 Jun 2023 21:43:44 +0800
Subject: [PATCH 382/404] [fix][broker] Support getStats/update partitioned
 topic with `-partition-` (#19235)

---
 .../pulsar/broker/admin/v2/NonPersistentTopics.java  |  6 +++++-
 .../pulsar/broker/admin/v2/PersistentTopics.java     | 12 ++++++++++--
 .../PartitionKeywordCompatibilityTest.java           |  3 +++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
index cc269d02831d8..a21aff3b365a2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java
@@ -217,7 +217,11 @@ public void getPartitionedStats(
             @ApiParam(value = "If return the earliest time in backlog")
             @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog) {
         try {
-            validatePartitionedTopicName(tenant, namespace, encodedTopic);
+            validateTopicName(tenant, namespace, encodedTopic);
+            if (topicName.isPartitioned()) {
+                throw new RestException(Response.Status.PRECONDITION_FAILED,
+                        "Partitioned Topic Name should not contain '-partition-'");
+            }
             if (topicName.isGlobal()) {
                 try {
                     validateGlobalNamespaceOwnership(namespaceName);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index eee363aeed86b..0fac3a8e005fc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -810,7 +810,11 @@ public void updatePartitionedTopic(
             @ApiParam(value = "The number of partitions for the topic",
                     required = true, type = "int", defaultValue = "0")
                     int numPartitions) {
-        validatePartitionedTopicName(tenant, namespace, encodedTopic);
+        validateTopicName(tenant, namespace, encodedTopic);
+        if (topicName.isPartitioned()) {
+            throw new RestException(Response.Status.PRECONDITION_FAILED,
+                    "Partitioned Topic Name should not contain '-partition-'");
+        }
         validateTopicPolicyOperationAsync(topicName, PolicyName.PARTITION, PolicyOperation.WRITE)
                 .thenCompose(__ -> internalUpdatePartitionedTopicAsync(numPartitions, updateLocalTopic, force))
                 .thenAccept(__ -> {
@@ -1286,7 +1290,11 @@ public void getPartitionedStats(
             @ApiParam(value = "If return the earliest time in backlog")
             @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog) {
         try {
-            validatePartitionedTopicName(tenant, namespace, encodedTopic);
+            validateTopicName(tenant, namespace, encodedTopic);
+            if (topicName.isPartitioned()) {
+                throw new RestException(Response.Status.PRECONDITION_FAILED,
+                        "Partitioned Topic Name should not contain '-partition-'");
+            }
             internalGetPartitionedStats(asyncResponse, authoritative, perPartition, getPreciseBacklog,
                     subscriptionBacklogSize, getEarliestTimeInBacklog);
         } catch (WebApplicationException wae) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java
index 86a5fcdc05aec..3eabb12b88921 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java
@@ -28,6 +28,7 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
+import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -70,6 +71,8 @@ public void testAutoCreatePartitionTopicWithKeywordAndDeleteIt()
         Assert.assertTrue(topics.contains(TopicName.get(topicName).getPartition(0).toString()));
         Assert.assertTrue(partitionedTopicList.contains(topicName));
         consumer.close();
+        PartitionedTopicStats stats = admin.topics().getPartitionedStats(topicName, false);
+        Assert.assertEquals(stats.getSubscriptions().size(), 1);
         admin.topics().deletePartitionedTopic(topicName);
         topics = admin.topics().getList("public/default");
         partitionedTopicList = admin.topics().getPartitionedTopicList("public/default");

From 0a39b819a9ec371322bfde61685497a2c21d9ab3 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 6 Jun 2023 12:06:05 -0500
Subject: [PATCH 383/404] [fix][fn] JavaInstanceStarter --tls_allow_insecure
 default to false (#20267)

---
 .../apache/pulsar/functions/runtime/JavaInstanceStarter.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index 33e837d66e250..89281a2f550e2 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -99,7 +99,7 @@ public class JavaInstanceStarter implements AutoCloseable {
     public String useTls = Boolean.FALSE.toString();
 
     @Parameter(names = "--tls_allow_insecure", description = "Allow insecure tls connection\n")
-    public String tlsAllowInsecureConnection = Boolean.TRUE.toString();
+    public String tlsAllowInsecureConnection = Boolean.FALSE.toString();
 
     @Parameter(names = "--hostname_verification_enabled", description = "Enable hostname verification")
     public String tlsHostNameVerificationEnabled = Boolean.FALSE.toString();

From 50b9a93e42e412d9f17b1637287d1a4c7c7ab148 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 6 Jun 2023 21:46:46 -0500
Subject: [PATCH 384/404] [fix][fn] TLS args admin download command use zero
 arity (#20513)

### Motivation

#20482 broke the function download command with this error:

```
Expected a command, got false
Usage: pulsar-admin [options] [command] [command options]
  Options:
    --admin-url
      Admin Service URL to which to connect.
      Default: http://localhost:8080/
```

The problem is that the TLS args for hostname verification and for insecure TLS are zero arity, and therefore, we should not add the `false` or the `true` arguments.

### Modifications

* Correct the changes made to the TLS args passed to the `pulsar-admin` CLI tool

### Documentation

- [x] `doc-not-needed`
---
 .../runtime/kubernetes/KubernetesRuntime.java         | 11 ++++++-----
 .../runtime/kubernetes/KubernetesRuntimeTest.java     |  4 +---
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
index 8d7a95cf3d71a..d0e36ecb48cd6 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
@@ -885,11 +885,12 @@ && isNotBlank(authConfig.getClientAuthenticationParameters())) {
                         "--auth-params",
                         authConfig.getClientAuthenticationParameters()));
             }
-            cmd.addAll(Arrays.asList(
-                    "--tls-allow-insecure",
-                    Boolean.toString(authConfig.isTlsAllowInsecureConnection()),
-                    "--tls-enable-hostname-verification",
-                    Boolean.toString(authConfig.isTlsHostnameVerificationEnable())));
+            if (authConfig.isTlsAllowInsecureConnection()) {
+                cmd.add("--tls-allow-insecure");
+            }
+            if (authConfig.isTlsHostnameVerificationEnable()) {
+                cmd.add("--tls-enable-hostname-verification");
+            }
             if (isNotBlank(authConfig.getTlsTrustCertsFilePath())) {
                 cmd.addAll(Arrays.asList(
                         "--tls-trust-cert-path",
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index 2cbbeb8a00b92..31f82adfe8c26 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -852,7 +852,6 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception {
         V1StatefulSet spec = container.createStatefulSet();
         String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
                 + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
-                + " --tls-allow-insecure false --tls-enable-hostname-verification false"
                 + " functions download "
                 + "--tenant " + TEST_TENANT
                 + " --namespace " + TEST_NAMESPACE
@@ -879,7 +878,6 @@ public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws
         V1StatefulSet spec = container.createStatefulSet();
         String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
                 + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
-                + " --tls-allow-insecure false --tls-enable-hostname-verification false"
                 + " functions download "
                 + "--tenant " + TEST_TENANT
                 + " --namespace " + TEST_NAMESPACE
@@ -909,7 +907,7 @@ public void testCustomKubernetesDownloadCommandsWithAuthAndCustomTLSWithoutAuthS
         V1StatefulSet spec = container.createStatefulSet();
         String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl
                 + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}"
-                + " --tls-allow-insecure false --tls-enable-hostname-verification true"
+                + " --tls-enable-hostname-verification"
                 + " --tls-trust-cert-path /my/ca.pem"
                 + " functions download "
                 + "--tenant " + TEST_TENANT

From f53cdda4599460ee364a4883a120ece46fe96ac2 Mon Sep 17 00:00:00 2001
From: "Kim, Joo Hyuk" 
Date: Wed, 7 Jun 2023 16:27:20 +0900
Subject: [PATCH 385/404] [fix][admin] Make failed `bin/pulsar-admin source`
 command exit with code `1 (failed)`  instead of `0 (success)` (#20503)

---
 .../pulsar/admin/cli/PulsarAdminToolTest.java  | 18 ++++++++++++++++++
 .../apache/pulsar/admin/cli/CmdSources.java    |  4 +---
 .../pulsar/admin/cli/TestCmdSources.java       | 15 +++++++++++++++
 3 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
index a1d5d695c6398..282336b4c37d3 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java
@@ -2204,6 +2204,24 @@ public void requestTimeout() throws Exception {
         assertEquals(1000, conf.getRequestTimeoutMs());
     }
 
+    @Test
+    public void testSourceCreateMissingSourceConfigFileFaileWithExitCode1() throws Exception {
+        Properties properties = new Properties();
+        properties.put("webServiceUrl", "http://localhost:2181");
+        PulsarAdminTool tool = new PulsarAdminTool(properties);
+
+        assertFalse(tool.run("sources create --source-config-file doesnotexist.yaml".split(" ")));
+    }
+
+    @Test
+    public void testSourceUpdateMissingSourceConfigFileFaileWithExitCode1() throws Exception {
+        Properties properties = new Properties();
+        properties.put("webServiceUrl", "http://localhost:2181");
+        PulsarAdminTool tool = new PulsarAdminTool(properties);
+
+        assertFalse(tool.run("sources update --source-config-file doesnotexist.yaml".split(" ")));
+    }
+
     @Test
     public void testAuthTlsWithJsonParam() throws Exception {
 
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
index 3a6b15caf8ea8..be2e03206021b 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
@@ -115,11 +115,9 @@ void run() throws Exception {
             try {
                 processArguments();
             } catch (Exception e) {
-                System.err.println(e.getMessage());
-                System.err.println();
                 String chosenCommand = jcommander.getParsedCommand();
                 getUsageFormatter().usage(chosenCommand);
-                return;
+                throw e;
             }
             runCmd();
         }
diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java
index ed90748f40f75..7d9a0a1f50c18 100644
--- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java
+++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java
@@ -25,6 +25,7 @@
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertTrue;
 
 import com.beust.jcommander.ParameterException;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -35,6 +36,7 @@
 import java.nio.file.Files;
 import java.util.Map;
 
+import java.util.UUID;
 import org.apache.pulsar.admin.cli.utils.CmdUtils;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.admin.Sources;
@@ -444,6 +446,19 @@ public void testCmdSourceConfigFile(SourceConfig testSourceConfig, SourceConfig
         verify(localSourceRunner).validateSourceConfigs(eq(expectedSourceConfig));
     }
 
+    @Test
+    public void testCmdSourcesThrowingExceptionOnFailure() throws Exception {
+        verifyNoSuchFileParameterException(createSource);
+        verifyNoSuchFileParameterException(updateSource);
+        verifyNoSuchFileParameterException(localSourceRunner);
+    }
+    
+    private void verifyNoSuchFileParameterException(org.apache.pulsar.admin.cli.CmdSources.SourceDetailsCommand command) {
+        command.sourceConfigFile = UUID.randomUUID().toString();
+        ParameterException e = Assert.expectThrows(ParameterException.class, command::processArguments);
+        assertTrue(e.getMessage().endsWith("(No such file or directory)"));
+    }
+    
 
     @Test
     public void testCliOverwriteConfigFile() throws Exception {

From d7186a67fc13ae972a76fbd9ba59b96d8bc7daae Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 7 Jun 2023 18:38:18 +0800
Subject: [PATCH 386/404] [fix][ml] There are two same-named managed ledgers in
 the one broker (#18688)

---
 .../impl/ManagedLedgerFactoryImpl.java        | 16 +++++--
 .../mledger/impl/ManagedLedgerImpl.java       |  9 ++++
 .../impl/ManagedLedgerFactoryTest.java        | 43 +++++++++++++++++++
 3 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index f076f68299dd0..9107b76c88a28 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -505,9 +505,19 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob
     }
 
     void close(ManagedLedger ledger) {
-        // Remove the ledger from the internal factory cache
-        ledgers.remove(ledger.getName());
-        entryCacheManager.removeEntryCache(ledger.getName());
+        // If the future in map is not done or has exceptionally complete, it means that @param-ledger is not in the
+        // map.
+        CompletableFuture ledgerFuture = ledgers.get(ledger.getName());
+        if (ledgerFuture == null || !ledgerFuture.isDone() || ledgerFuture.isCompletedExceptionally()){
+            return;
+        }
+        if (ledgerFuture.join() != ledger){
+            return;
+        }
+        // Remove the ledger from the internal factory cache.
+        if (ledgers.remove(ledger.getName(), ledgerFuture)) {
+            entryCacheManager.removeEntryCache(ledger.getName());
+        }
     }
 
     public CompletableFuture shutdownAsync() throws ManagedLedgerException {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 9b3d7e46aaa8c..10f7948f553cb 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1528,6 +1528,15 @@ private void closeAllCursors(CloseCallback callback, final Object ctx) {
 
     @Override
     public synchronized void createComplete(int rc, final LedgerHandle lh, Object ctx) {
+        if (STATE_UPDATER.get(this) == State.Closed) {
+            if (lh != null) {
+                log.warn("[{}] ledger create completed after the managed ledger is closed rc={} ledger={}, so just"
+                        + " close this ledger handle.", name, rc, lh != null ? lh.getId() : -1);
+                lh.closeAsync();
+            }
+            return;
+        }
+
         if (log.isDebugEnabled()) {
             log.debug("[{}] createComplete rc={} ledger={}", name, rc, lh != null ? lh.getId() : -1);
         }
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java
index d49d9ab3e2b6b..8695759c99f62 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java
@@ -20,12 +20,16 @@
 
 import static org.testng.Assert.assertEquals;
 
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo.CursorInfo;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo.MessageRangeInfo;
 import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
 public class ManagedLedgerFactoryTest extends MockedBookKeeperTestCase {
@@ -71,4 +75,43 @@ public void testGetManagedLedgerInfoWithClose() throws Exception {
         assertEquals(mri.to.entryId, 0);
     }
 
+    /**
+     * see: https://github.com/apache/pulsar/pull/18688
+     */
+    @Test
+    public void testConcurrentCloseLedgerAndSwitchLedgerForReproduceIssue() throws Exception {
+        String managedLedgerName = "lg_" + UUID.randomUUID().toString().replaceAll("-", "_");
+
+        ManagedLedgerConfig config = new ManagedLedgerConfig();
+        config.setThrottleMarkDelete(1);
+        config.setMaximumRolloverTime(Integer.MAX_VALUE, TimeUnit.SECONDS);
+        config.setMaxEntriesPerLedger(5);
+
+        // create managedLedger once and close it.
+        ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) factory.open(managedLedgerName, config);
+        waitManagedLedgerStateEquals(managedLedger1, ManagedLedgerImpl.State.LedgerOpened);
+        managedLedger1.close();
+
+        // create managedLedger the second time.
+        ManagedLedgerImpl managedLedger2 = (ManagedLedgerImpl) factory.open(managedLedgerName, config);
+        waitManagedLedgerStateEquals(managedLedger2, ManagedLedgerImpl.State.LedgerOpened);
+
+        // Mock the task create ledger complete now, it will change the state to another value which not is Closed.
+        // Close managedLedger1 the second time.
+        managedLedger1.createComplete(1, null, null);
+        managedLedger1.close();
+
+        // Verify managedLedger2 is still there.
+        Assert.assertFalse(factory.ledgers.isEmpty());
+        Assert.assertEquals(factory.ledgers.get(managedLedger2.getName()).join(), managedLedger2);
+
+        // cleanup.
+        managedLedger2.close();
+    }
+
+    private void waitManagedLedgerStateEquals(ManagedLedgerImpl managedLedger, ManagedLedgerImpl.State expectedStat){
+        Awaitility.await().untilAsserted(() ->
+                Assert.assertTrue(managedLedger.getState() == expectedStat));
+    }
+
 }

From 03f916702ec2a887833543fdea5a2d5305a87302 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 7 Jun 2023 16:39:45 +0300
Subject: [PATCH 387/404] [fix][broker] Restore solution for certain topic
 unloading race conditions (#20527)

---
 .../java/org/apache/pulsar/broker/service/BrokerService.java  | 4 ----
 .../pulsar/broker/service/persistent/PersistentTopic.java     | 2 +-
 .../apache/pulsar/broker/service/BrokerBkEnsemblesTests.java  | 4 ++--
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index d952f7d65743a..c38514237856b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -2227,10 +2227,6 @@ public AuthorizationService getAuthorizationService() {
         return authorizationService;
     }
 
-    public CompletableFuture removeTopicFromCache(String topicName) {
-        return removeTopicFutureFromCache(topicName, null);
-    }
-
     public CompletableFuture removeTopicFromCache(Topic topic) {
         Optional>> createTopicFuture = findTopicFutureInCache(topic);
         if (createTopicFuture.isEmpty()){
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 98e51a2e3ed6e..2a0c229daf6c2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -1507,7 +1507,7 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) {
     }
 
     private void disposeTopic(CompletableFuture closeFuture) {
-        brokerService.removeTopicFromCache(topic)
+        brokerService.removeTopicFromCache(PersistentTopic.this)
                 .thenRun(() -> {
                     replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java
index e69714e539be6..9f19bda3647f3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java
@@ -127,7 +127,7 @@ public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception {
         // (3) remove topic and managed-ledger from broker which means topic is not closed gracefully
         consumer.close();
         producer.close();
-        pulsar.getBrokerService().removeTopicFromCache(topic1);
+        pulsar.getBrokerService().removeTopicFromCache(topic);
         ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory();
         Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
         field.setAccessible(true);
@@ -252,7 +252,7 @@ public void testSkipCorruptDataLedger() throws Exception {
 
         // clean managed-ledger and recreate topic to clean any data from the cache
         producer.close();
-        pulsar.getBrokerService().removeTopicFromCache(topic1);
+        pulsar.getBrokerService().removeTopicFromCache(topic);
         ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory();
         Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
         field.setAccessible(true);

From 60dba5d675f998a5108a35be65649edd57ce8596 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 7 Jun 2023 20:25:03 +0300
Subject: [PATCH 388/404] [fix][test] Reduce flakiness of AdminApi2Test
 (#20529)

---
 .../pulsar/broker/admin/AdminApi2Test.java    | 397 ++++++++++--------
 .../broker/testcontext/PulsarTestContext.java |   4 +
 2 files changed, 237 insertions(+), 164 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java
index 7ed5fe34ea4f7..26449b69edd55 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.admin;
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -58,7 +59,6 @@
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.commons.lang3.reflect.FieldUtils;
-import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService;
@@ -127,6 +127,7 @@
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
@@ -135,6 +136,10 @@
 public class AdminApi2Test extends MockedPulsarServiceBaseTest {
 
     private MockedPulsarService mockPulsarSetup;
+    private boolean restartClusterAfterTest;
+    private int usageCount;
+    private String defaultNamespace;
+    private String defaultTenant;
 
     @BeforeClass
     @Override
@@ -152,10 +157,17 @@ public void setup() throws Exception {
     @Override
     protected ServiceConfiguration getDefaultConf() {
         ServiceConfiguration conf = super.getDefaultConf();
+        configureDefaults(conf);
+        return conf;
+    }
+
+    void configureDefaults(ServiceConfiguration conf) {
         conf.setForceDeleteNamespaceAllowed(true);
         conf.setLoadBalancerEnabled(true);
         conf.setEnableNamespaceIsolationUpdateOnTime(true);
-        return conf;
+        conf.setAllowOverrideEntryFilters(true);
+        conf.setEntryFilterNames(List.of());
+        conf.setMaxNumPartitionsPerPartitionedTopic(0);
     }
 
     @AfterClass(alwaysRun = true)
@@ -171,6 +183,20 @@ public void cleanup() throws Exception {
 
     @AfterMethod(alwaysRun = true)
     public void resetClusters() throws Exception {
+        if (restartClusterAfterTest) {
+            restartClusterAndResetUsageCount();
+        } else {
+            try {
+                cleanupCluster();
+            } catch (Exception e) {
+                log.error("Failed to clean up state by deleting namespaces and tenants after test. "
+                        + "Restarting the test broker.", e);
+                restartClusterAndResetUsageCount();
+            }
+        }
+    }
+
+    private void cleanupCluster() throws Exception {
         pulsar.getConfiguration().setForceDeleteTenantAllowed(true);
         pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true);
         for (String tenant : admin.tenants().getTenants()) {
@@ -178,22 +204,59 @@ public void resetClusters() throws Exception {
                 deleteNamespaceWithRetry(namespace, true, admin, pulsar,
                         mockPulsarSetup.getPulsar());
             }
-            admin.tenants().deleteTenant(tenant, true);
+            try {
+                admin.tenants().deleteTenant(tenant, true);
+            } catch (Exception e) {
+                log.error("Failed to delete tenant {} after test", tenant, e);
+                String zkDirectory = "/managed-ledgers/" + tenant;
+                try {
+                    log.info("Listing {} to see if existing keys are preventing deletion.", zkDirectory);
+                    pulsar.getPulsarResources().getLocalMetadataStore().get().getChildren(zkDirectory)
+                            .get(5, TimeUnit.SECONDS).forEach(key -> log.info("Child key '{}'", key));
+                } catch (Exception ignore) {
+                    log.error("Failed to list tenant {} ZK directory {} after test", tenant, zkDirectory, e);
+                }
+                throw e;
+            }
         }
 
         for (String cluster : admin.clusters().getClusters()) {
             admin.clusters().deleteCluster(cluster);
         }
 
-        resetConfig();
+        configureDefaults(conf);
         setupClusters();
     }
 
+    private void restartClusterAfterTest() {
+        restartClusterAfterTest = true;
+    }
+
+    private void restartClusterAndResetUsageCount() throws Exception {
+        cleanup();
+        restartClusterAfterTest = false;
+        usageCount = 0;
+        setup();
+    }
+
+    private void restartClusterIfReused() throws Exception {
+        if (usageCount > 1) {
+            restartClusterAndResetUsageCount();
+        }
+    }
+
+    @BeforeMethod
+    public void increaseUsageCount() {
+        usageCount++;
+    }
+
     private void setupClusters() throws PulsarAdminException {
         admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
-        admin.tenants().createTenant("prop-xyz", tenantInfo);
-        admin.namespaces().createNamespace("prop-xyz/ns1", Set.of("test"));
+        defaultTenant = newUniqueName("prop-xyz");
+        admin.tenants().createTenant(defaultTenant, tenantInfo);
+        defaultNamespace = defaultTenant + "/ns1";
+        admin.namespaces().createNamespace(defaultNamespace, Set.of("test"));
     }
 
     @DataProvider(name = "topicType")
@@ -250,7 +313,7 @@ public void testIncrementPartitionsOfTopic() throws Exception {
         final String subName2 = topicName + "-my-sub-2/encode";
         final int startPartitions = 4;
         final int newPartitions = 8;
-        final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName;
+        final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + topicName;
 
         URL pulsarUrl = new URL(pulsar.getWebServiceAddress());
 
@@ -299,7 +362,7 @@ public void testIncrementPartitionsOfTopic() throws Exception {
         assertEquals(new HashSet<>(admin.topics().getSubscriptions(newPartitionTopicName)),
                 Set.of(subName1, subName2));
 
-        assertEquals(new HashSet<>(admin.topics().getList("prop-xyz/ns1")).size(), newPartitions);
+        assertEquals(new HashSet<>(admin.topics().getList(defaultNamespace)).size(), newPartitions);
 
         // test cumulative stats for partitioned topic
         PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
@@ -333,14 +396,15 @@ public void testIncrementPartitionsOfTopic() throws Exception {
 
     @Test
     public void testTopicPoliciesWithMultiBroker() throws Exception {
+        restartClusterAfterTest();
+
         //setup cluster with 3 broker
-        cleanup();
-        setup();
         admin.clusters().updateCluster("test",
                 ClusterData.builder().serviceUrl((pulsar.getWebServiceAddress() + ",localhost:1026," + "localhost:2050")).build());
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
-        admin.tenants().createTenant("prop-xyz2", tenantInfo);
-        admin.namespaces().createNamespace("prop-xyz2/ns1", Set.of("test"));
+        String tenantName = newUniqueName("prop-xyz2");
+        admin.tenants().createTenant(tenantName, tenantInfo);
+        admin.namespaces().createNamespace(tenantName + "/ns1", Set.of("test"));
         conf.setBrokerServicePort(Optional.of(1024));
         conf.setBrokerServicePortTls(Optional.of(1025));
         conf.setWebServicePort(Optional.of(1026));
@@ -361,14 +425,14 @@ public void testTopicPoliciesWithMultiBroker() throws Exception {
         PulsarAdmin admin3 = PulsarAdmin.builder().serviceHttpUrl(pulsar3.getWebServiceAddress()).build();
 
         //for partitioned topic, we can get topic policies from every broker
-        final String topic = "persistent://prop-xyz2/ns1/" + BrokerTestUtil.newUniqueName("test");
+        final String topic = "persistent://" + tenantName + "/ns1/" + newUniqueName("test");
         int partitionNum = 3;
         admin.topics().createPartitionedTopic(topic, partitionNum);
         pulsarClient.newConsumer().topic(topic).subscriptionName("sub").subscribe().close();
 
         setTopicPoliciesAndValidate(admin2, admin3, topic);
         //for non-partitioned topic, we can get topic policies from every broker
-        final String topic2 = "persistent://prop-xyz2/ns1/" + BrokerTestUtil.newUniqueName("test");
+        final String topic2 = "persistent://" + tenantName + "/ns1/" + newUniqueName("test");
         pulsarClient.newConsumer().topic(topic2).subscriptionName("sub").subscribe().close();
         setTopicPoliciesAndValidate(admin2, admin3, topic2);
     }
@@ -402,7 +466,7 @@ private void setTopicPoliciesAndValidate(PulsarAdmin admin2
     public void nonPersistentTopics() throws Exception {
         final String topicName = "nonPersistentTopic";
 
-        final String nonPersistentTopicName = "non-persistent://prop-xyz/ns1/" + topicName;
+        final String nonPersistentTopicName = "non-persistent://" + defaultNamespace + "/" + topicName;
         // Force to create a topic
         publishMessagesOnTopic(nonPersistentTopicName, 0, 0);
 
@@ -434,7 +498,7 @@ public void nonPersistentTopics() throws Exception {
         assertFalse(topicStats.getSubscriptions().containsKey("my-sub"));
         assertEquals(topicStats.getPublishers().size(), 0);
         // test partitioned-topic
-        final String partitionedTopicName = "non-persistent://prop-xyz/ns1/paritioned";
+        final String partitionedTopicName = "non-persistent://" + defaultNamespace + "/paritioned";
         admin.topics().createPartitionedTopic(partitionedTopicName, 5);
         assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 5);
     }
@@ -462,7 +526,7 @@ private void publishMessagesOnTopic(String topicName, int messages, int startIdx
     @Test
     public void testSetPersistencePolicies() throws Exception {
 
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         admin.namespaces().createNamespace(namespace, Set.of("test"));
 
         assertNull(admin.namespaces().getPersistence(namespace));
@@ -500,7 +564,7 @@ public void testSetPersistencePolicies() throws Exception {
     @Test
     public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
 
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String topicName = "persistent://" + namespace + "/topic1";
         admin.namespaces().createNamespace(namespace, Set.of("test"));
 
@@ -541,7 +605,7 @@ public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
     @Test(dataProvider = "topicType")
     public void testUnloadTopic(final String topicType) throws Exception {
 
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String topicName = topicType + "://" + namespace + "/topic1";
         admin.namespaces().createNamespace(namespace, Set.of("test"));
 
@@ -591,11 +655,12 @@ private void unloadTopic(String topicName) throws Exception {
      */
     @Test(dataProvider = "namespaceNames", timeOut = 30000)
     public void testResetCursorOnPosition(String namespaceName) throws Exception {
-        final String topicName = "persistent://prop-xyz/use/" + namespaceName + "/resetPosition";
+        restartClusterAfterTest();
+        final String topicName = "persistent://" + defaultTenant + "/use/" + namespaceName + "/resetPosition";
         final int totalProducedMessages = 50;
 
         // set retention
-        admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));
+        admin.namespaces().setRetention(defaultNamespace, new RetentionPolicies(10, 10));
 
         // create consumer and subscription
         Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
@@ -674,14 +739,11 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception {
         }
 
         consumer.close();
-
-        cleanup();
-        setup();
     }
 
     @Test
     public void shouldNotSupportResetOnPartitionedTopic() throws PulsarAdminException, PulsarClientException {
-        final String partitionedTopicName = "persistent://prop-xyz/ns1/" + BrokerTestUtil.newUniqueName("parttopic");
+        final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + newUniqueName("parttopic");
         admin.topics().createPartitionedTopic(partitionedTopicName, 4);
         @Cleanup
         Consumer consumer = pulsarClient.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub")
@@ -713,7 +775,9 @@ private void publishMessagesOnPersistentTopic(String topicName, int messages, in
 
     @Test(timeOut = 20000)
     public void testMaxConsumersOnSubApi() throws Exception {
-        final String namespace = "prop-xyz/ns1";
+        final String namespace = newUniqueName(defaultTenant + "/ns");
+        admin.namespaces().createNamespace(namespace, Set.of("test"));
+
         assertNull(admin.namespaces().getMaxConsumersPerSubscription(namespace));
         admin.namespaces().setMaxConsumersPerSubscription(namespace, 10);
         Awaitility.await().untilAsserted(() -> {
@@ -732,7 +796,7 @@ public void testMaxConsumersOnSubApi() throws Exception {
      */
     @Test
     public void testLoadReportApi() throws Exception {
-
+        restartClusterAfterTest();
         this.conf.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
         @Cleanup("cleanup")
         MockedPulsarService mockPulsarSetup1 = new MockedPulsarService(this.conf);
@@ -795,6 +859,8 @@ public void testPeerCluster() throws Exception {
      */
     @Test
     public void testReplicationPeerCluster() throws Exception {
+        restartClusterAfterTest();
+
         admin.clusters().createCluster("us-west1",
                 ClusterData.builder().serviceUrl("http://broker.messaging.west1.example.com:8080").build());
         admin.clusters().createCluster("us-west2",
@@ -814,7 +880,7 @@ public void testReplicationPeerCluster() throws Exception {
         assertEquals(allClusters,
                 List.of("test", "us-east1", "us-east2", "us-west1", "us-west2", "us-west3", "us-west4"));
 
-        final String property = "peer-prop";
+        final String property = newUniqueName("peer-prop");
         Set allowedClusters = Set.of("us-west1", "us-west2", "us-west3", "us-west4", "us-east1",
                 "us-east2", "global");
         TenantInfoImpl propConfig = new TenantInfoImpl(Set.of("test"), allowedClusters);
@@ -848,9 +914,6 @@ public void testReplicationPeerCluster() throws Exception {
         clusterIds = Set.of("us-west1", "us-west4");
         // no peer coexist in replication clusters
         admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds);
-
-        cleanup();
-        setup();
     }
 
     @Test
@@ -889,7 +952,7 @@ public void clusterFailureDomain() throws PulsarAdminException {
 
     @Test
     public void namespaceAntiAffinity() throws PulsarAdminException {
-        final String namespace = "prop-xyz/ns1";
+        final String namespace = defaultNamespace;
         final String antiAffinityGroup = "group";
         assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace)));
         admin.namespaces().setNamespaceAntiAffinityGroup(namespace, antiAffinityGroup);
@@ -897,9 +960,9 @@ public void namespaceAntiAffinity() throws PulsarAdminException {
         admin.namespaces().deleteNamespaceAntiAffinityGroup(namespace);
         assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace)));
 
-        final String ns1 = "prop-xyz/antiAG1";
-        final String ns2 = "prop-xyz/antiAG2";
-        final String ns3 = "prop-xyz/antiAG3";
+        final String ns1 = defaultTenant + "/antiAG1";
+        final String ns2 = defaultTenant + "/antiAG2";
+        final String ns3 = defaultTenant + "/antiAG3";
         admin.namespaces().createNamespace(ns1, Set.of("test"));
         admin.namespaces().createNamespace(ns2, Set.of("test"));
         admin.namespaces().createNamespace(ns3, Set.of("test"));
@@ -908,19 +971,19 @@ public void namespaceAntiAffinity() throws PulsarAdminException {
         admin.namespaces().setNamespaceAntiAffinityGroup(ns3, antiAffinityGroup);
 
         Set namespaces = new HashSet<>(
-                admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", antiAffinityGroup));
+                admin.namespaces().getAntiAffinityNamespaces(defaultTenant, "test", antiAffinityGroup));
         assertEquals(namespaces.size(), 3);
         assertTrue(namespaces.contains(ns1));
         assertTrue(namespaces.contains(ns2));
         assertTrue(namespaces.contains(ns3));
 
-        List namespaces2 = admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", "invalid-group");
+        List namespaces2 = admin.namespaces().getAntiAffinityNamespaces(defaultTenant, "test", "invalid-group");
         assertEquals(namespaces2.size(), 0);
     }
 
     @Test
     public void testPersistentTopicList() throws Exception {
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String topicName = "non-persistent://" + namespace + "/bundle-topic";
         admin.namespaces().createNamespace(namespace, 20);
         admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of("test"));
@@ -954,7 +1017,7 @@ public void testPersistentTopicList() throws Exception {
 
     @Test
     public void testCreateAndGetTopicProperties() throws Exception {
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String nonPartitionedTopicName = "persistent://" + namespace + "/non-partitioned-TopicProperties";
         admin.namespaces().createNamespace(namespace, 20);
         Map nonPartitionedTopicProperties = new HashMap<>();
@@ -975,7 +1038,7 @@ public void testCreateAndGetTopicProperties() throws Exception {
 
     @Test
     public void testUpdatePartitionedTopicProperties() throws Exception {
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String topicName = "persistent://" + namespace + "/testUpdatePartitionedTopicProperties";
         final String topicNameTwo = "persistent://" + namespace + "/testUpdatePartitionedTopicProperties2";
         admin.namespaces().createNamespace(namespace, 20);
@@ -1033,7 +1096,7 @@ public void testUpdatePartitionedTopicProperties() throws Exception {
 
     @Test
     public void testUpdateNonPartitionedTopicProperties() throws Exception {
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String topicName = "persistent://" + namespace + "/testUpdateNonPartitionedTopicProperties";
         admin.namespaces().createNamespace(namespace, 20);
 
@@ -1068,7 +1131,7 @@ public void testUpdateNonPartitionedTopicProperties() throws Exception {
 
     @Test
     public void testNonPersistentTopics() throws Exception {
-        final String namespace = "prop-xyz/ns2";
+        final String namespace = newUniqueName(defaultTenant + "/ns2");
         final String topicName = "non-persistent://" + namespace + "/topic";
         admin.namespaces().createNamespace(namespace, 20);
         admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of("test"));
@@ -1096,7 +1159,7 @@ public void testNonPersistentTopics() throws Exception {
     public void testPublishConsumerStats() throws Exception {
         final String topicName = "statTopic";
         final String subscriberName = topicName + "-my-sub-1";
-        final String topic = "persistent://prop-xyz/ns1/" + topicName;
+        final String topic = "persistent://" + defaultNamespace + "/" + topicName;
         final String producerName = "myProducer";
 
         @Cleanup
@@ -1143,6 +1206,8 @@ public void testPublishConsumerStats() throws Exception {
 
     @Test
     public void testTenantNameWithUnderscore() throws Exception {
+        restartClusterAfterTest();
+
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
         admin.tenants().createTenant("prop_xyz", tenantInfo);
 
@@ -1150,15 +1215,12 @@ public void testTenantNameWithUnderscore() throws Exception {
 
         String topic = "persistent://prop_xyz/use/my-namespace/my-topic";
 
+        @Cleanup
         Producer producer = pulsarClient.newProducer().topic(topic)
                 .create();
 
         TopicStats stats = admin.topics().getStats(topic);
         assertEquals(stats.getPublishers().size(), 1);
-        producer.close();
-
-        cleanup();
-        setup();
     }
 
     @Test
@@ -1200,11 +1262,11 @@ public void testTenantWithNonexistentClusters() throws Exception {
         assertFalse(admin.tenants().getTenants().contains("test-tenant"));
 
         // Check existing tenant
-        assertTrue(admin.tenants().getTenants().contains("prop-xyz"));
+        assertTrue(admin.tenants().getTenants().contains(defaultTenant));
 
         // If we try to update existing tenant with nonexistent clusters, it should fail immediately
         try {
-            admin.tenants().updateTenant("prop-xyz", tenantInfo);
+            admin.tenants().updateTenant(defaultTenant, tenantInfo);
             fail("Should have failed");
         } catch (PulsarAdminException e) {
             assertEquals(e.getStatusCode(), Status.PRECONDITION_FAILED.getStatusCode());
@@ -1264,7 +1326,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception {
     @Test
     public void brokerNamespaceIsolationPoliciesUpdateOnTime() throws Exception {
         String brokerName = pulsar.getAdvertisedAddress();
-        String ns1Name = "prop-xyz/test_ns1_iso_" + System.currentTimeMillis();
+        String ns1Name = defaultTenant + "/test_ns1_iso_" + System.currentTimeMillis();
         admin.namespaces().createNamespace(ns1Name, Set.of("test"));
 
         //  0. without isolation policy configured, lookup will success.
@@ -1335,15 +1397,16 @@ public void clustersList() throws PulsarAdminException {
      */
     @Test
     public void testClusterIsReadyBeforeCreateTopic() throws Exception {
+        restartClusterAfterTest();
         final String topicName = "partitionedTopic";
         final int partitions = 4;
-        final String persistentPartitionedTopicName = "persistent://prop-xyz/ns2/" + topicName;
-        final String NonPersistentPartitionedTopicName = "non-persistent://prop-xyz/ns2/" + topicName;
+        final String persistentPartitionedTopicName = "persistent://" + defaultTenant + "/ns2/" + topicName;
+        final String NonPersistentPartitionedTopicName = "non-persistent://" + defaultTenant + "/ns2/" + topicName;
 
-        admin.namespaces().createNamespace("prop-xyz/ns2");
+        admin.namespaces().createNamespace(defaultTenant + "/ns2");
         // By default the cluster will configure as configuration file. So the create topic operation
         // will never throw exception except there is no cluster.
-        admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns2", new HashSet());
+        admin.namespaces().setNamespaceReplicationClusters(defaultTenant + "/ns2", new HashSet());
 
         try {
             admin.topics().createPartitionedTopic(persistentPartitionedTopicName, partitions);
@@ -1356,15 +1419,12 @@ public void testClusterIsReadyBeforeCreateTopic() throws Exception {
             Assert.fail("should have failed due to Namespace does not have any clusters configured");
         } catch (PulsarAdminException.PreconditionFailedException ignored) {
         }
-
-        cleanup();
-        setup();
     }
 
     @Test
     public void testCreateNamespaceWithNoClusters() throws PulsarAdminException {
         String localCluster = pulsar.getConfiguration().getClusterName();
-        String namespace = "prop-xyz/test-ns-with-no-clusters";
+        String namespace = newUniqueName(defaultTenant + "/test-ns-with-no-clusters");
         admin.namespaces().createNamespace(namespace);
 
         // Global cluster, if there, should be omitted from the results
@@ -1377,7 +1437,7 @@ public void testConsumerStatsLastTimestamp() throws PulsarClientException, Pulsa
         long timestamp = System.currentTimeMillis();
         final String topicName = "consumer-stats-" + timestamp;
         final String subscribeName = topicName + "-test-stats-sub";
-        final String topic = "persistent://prop-xyz/ns1/" + topicName;
+        final String topic = "persistent://" + defaultNamespace + "/" + topicName;
         final String producerName = "producer-" + topicName;
 
         @Cleanup
@@ -1480,10 +1540,9 @@ public void testConsumerStatsLastTimestamp() throws PulsarClientException, Pulsa
 
     @Test(timeOut = 30000)
     public void testPreciseBacklog() throws Exception {
-        cleanup();
-        setup();
+        restartClusterIfReused();
 
-        final String topic = "persistent://prop-xyz/ns1/precise-back-log";
+        final String topic = "persistent://" + defaultNamespace + "/precise-back-log";
         final String subName = "sub-name";
 
         @Cleanup
@@ -1533,6 +1592,7 @@ public void testPreciseBacklog() throws Exception {
 
     @Test
     public void testDeleteTenant() throws Exception {
+        restartClusterAfterTest();
         // Disabled conf: systemTopicEnabled. see: https://github.com/apache/pulsar/pull/17070
         boolean originalSystemTopicEnabled = conf.isSystemTopicEnabled();
         if (originalSystemTopicEnabled) {
@@ -1542,7 +1602,7 @@ public void testDeleteTenant() throws Exception {
         }
         pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false);
 
-        String tenant = "test-tenant-1";
+        String tenant = newUniqueName("test-tenant-1");
         assertFalse(admin.tenants().getTenants().contains(tenant));
 
         // create tenant
@@ -1588,12 +1648,6 @@ public void testDeleteTenant() throws Exception {
         assertFalse(pulsar.getLocalMetadataStore().exists(partitionedTopicPath).join());
         assertFalse(pulsar.getLocalMetadataStore().exists(localPoliciesPath).join());
         assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join());
-        // Reset conf: systemTopicEnabled
-        if (originalSystemTopicEnabled) {
-            cleanup();
-            conf.setSystemTopicEnabled(true);
-            setup();
-        }
     }
 
     @Data
@@ -1628,13 +1682,14 @@ private void setNamespaceAttr(NamespaceAttr namespaceAttr){
 
     @Test(dataProvider = "namespaceAttrs")
     public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception {
+        restartClusterAfterTest();
+
         // Set conf.
         cleanup();
-        NamespaceAttr originalNamespaceAttr = markOriginalNamespaceAttr();
         setNamespaceAttr(namespaceAttr);
         setup();
 
-        String tenant = "test-tenant";
+        String tenant = newUniqueName("test-tenant");
         assertFalse(admin.tenants().getTenants().contains(tenant));
 
         // create tenant
@@ -1675,16 +1730,11 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception {
 
         final String bundleDataPath = "/loadbalance/bundle-data/" + namespace;
         assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join());
-
-        // Reset config
-        cleanup();
-        setNamespaceAttr(originalNamespaceAttr);
-        setup();
     }
 
     @Test
     public void testDeleteNamespaceWithTopicPolicies() throws Exception {
-        String tenant = "test-tenant";
+        String tenant = newUniqueName("test-tenant");
         assertFalse(admin.tenants().getTenants().contains(tenant));
 
         // create tenant
@@ -1732,7 +1782,7 @@ public void testDeleteNamespaceWithTopicPolicies() throws Exception {
 
     @Test(timeOut = 30000)
     public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminException, InterruptedException {
-        final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-" + UUID.randomUUID().toString();
+        final String topic = "persistent://" + defaultNamespace + "/precise-back-log-no-delayed-" + UUID.randomUUID().toString();
         final String subName = "sub-name";
 
         @Cleanup
@@ -1788,7 +1838,7 @@ public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminExce
 
     @Test
     public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException, PulsarAdminException {
-        final String topic = "persistent://prop-xyz/ns1/precise-back-log-for-partitioned-topic";
+        final String topic = "persistent://" + defaultNamespace + "/precise-back-log-for-partitioned-topic";
         admin.topics().createPartitionedTopic(topic, 2);
         final String subName = "sub-name";
 
@@ -1830,7 +1880,7 @@ public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException
 
     @Test(timeOut = 30000)
     public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientException, PulsarAdminException, InterruptedException {
-        final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-partitioned-topic";
+        final String topic = "persistent://" + defaultNamespace + "/precise-back-log-no-delayed-partitioned-topic";
         admin.topics().createPartitionedTopic(topic, 2);
         final String subName = "sub-name";
 
@@ -1884,7 +1934,8 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti
 
     @Test
     public void testMaxNumPartitionsPerPartitionedTopicSuccess() {
-        final String topic = "persistent://prop-xyz/ns1/max-num-partitions-per-partitioned-topic-success";
+        restartClusterAfterTest();
+        final String topic = "persistent://" + defaultNamespace + "/max-num-partitions-per-partitioned-topic-success";
         pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(3);
 
         try {
@@ -1899,7 +1950,8 @@ public void testMaxNumPartitionsPerPartitionedTopicSuccess() {
 
     @Test
     public void testMaxNumPartitionsPerPartitionedTopicFailure() {
-        final String topic = "persistent://prop-xyz/ns1/max-num-partitions-per-partitioned-topic-failure";
+        restartClusterAfterTest();
+        final String topic = "persistent://" + defaultNamespace + "/max-num-partitions-per-partitioned-topic-failure";
         pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(2);
 
         try {
@@ -1916,21 +1968,24 @@ public void testMaxNumPartitionsPerPartitionedTopicFailure() {
     @Test
     public void testListOfNamespaceBundles() throws Exception {
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
-        admin.tenants().createTenant("prop-xyz2", tenantInfo);
-        admin.namespaces().createNamespace("prop-xyz2/ns1", 10);
-        admin.namespaces().setNamespaceReplicationClusters("prop-xyz2/ns1", Set.of("test"));
-        admin.namespaces().createNamespace("prop-xyz2/test/ns2", 10);
-        assertEquals(admin.namespaces().getBundles("prop-xyz2/ns1").getNumBundles(), 10);
-        assertEquals(admin.namespaces().getBundles("prop-xyz2/test/ns2").getNumBundles(), 10);
+        String tenantName = newUniqueName("prop-xyz2");
+        admin.tenants().createTenant(tenantName, tenantInfo);
+        admin.namespaces().createNamespace(tenantName + "/ns1", 10);
+        admin.namespaces().setNamespaceReplicationClusters(tenantName + "/ns1", Set.of("test"));
+        admin.namespaces().createNamespace(tenantName + "/test/ns2", 10);
+        assertEquals(admin.namespaces().getBundles(tenantName + "/ns1").getNumBundles(), 10);
+        assertEquals(admin.namespaces().getBundles(tenantName + "/test/ns2").getNumBundles(), 10);
 
-        admin.namespaces().deleteNamespace("prop-xyz2/test/ns2");
+        admin.namespaces().deleteNamespace(tenantName + "/test/ns2");
     }
 
     @Test
     public void testForceDeleteNamespace() throws Exception {
-        final String namespaceName = "prop-xyz2/ns1";
+        restartClusterAfterTest();
+        String tenantName = newUniqueName("prop-xyz2");
+        final String namespaceName = tenantName + "/ns1";
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
-        admin.tenants().createTenant("prop-xyz2", tenantInfo);
+        admin.tenants().createTenant(tenantName, tenantInfo);
         admin.namespaces().createNamespace(namespaceName, 1);
         final String topic = "persistent://" + namespaceName + "/test" + UUID.randomUUID();
         pulsarClient.newProducer(Schema.DOUBLE).topic(topic).create().close();
@@ -1942,17 +1997,15 @@ public void testForceDeleteNamespace() throws Exception {
         } catch (PulsarAdminException e) {
             assertEquals(e.getStatusCode(), 404);
         }
-
-        cleanup();
-        setup();
     }
 
     @Test
     public void testForceDeleteNamespaceWithAutomaticTopicCreation() throws Exception {
         conf.setForceDeleteNamespaceAllowed(true);
-        final String namespaceName = "prop-xyz2/ns1";
+        String tenantName = newUniqueName("prop-xyz2");
+        final String namespaceName = tenantName + "/ns1";
         TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test"));
-        admin.tenants().createTenant("prop-xyz2", tenantInfo);
+        admin.tenants().createTenant(tenantName, tenantInfo);
         admin.namespaces().createNamespace(namespaceName, 1);
         admin.namespaces().setAutoTopicCreation(namespaceName,
                 AutoTopicCreationOverride.builder()
@@ -1977,7 +2030,7 @@ public void testForceDeleteNamespaceWithAutomaticTopicCreation() throws Exceptio
             // the consumer will try to re-create the partitions
             admin.namespaces().deleteNamespace(namespaceName, true);
 
-            assertFalse(admin.namespaces().getNamespaces("prop-xyz2").contains("ns1"));
+            assertFalse(admin.namespaces().getNamespaces(tenantName).contains("ns1"));
         }
     }
 
@@ -2000,6 +2053,7 @@ public void testUpdateClusterWithProxyUrl() throws Exception {
 
     @Test
     public void testMaxNamespacesPerTenant() throws Exception {
+        restartClusterAfterTest();
         cleanup();
         conf.setMaxNamespacesPerTenant(2);
         setup();
@@ -2013,19 +2067,11 @@ public void testMaxNamespacesPerTenant() throws Exception {
             Assert.assertEquals(e.getStatusCode(), 412);
             Assert.assertEquals(e.getHttpError(), "Exceed the maximum number of namespace in tenant :testTenant");
         }
-
-        //unlimited
-        cleanup();
-        conf.setMaxNamespacesPerTenant(0);
-        setup();
-        admin.tenants().createTenant("testTenant", tenantInfo);
-        for (int i = 0; i < 10; i++) {
-            admin.namespaces().createNamespace("testTenant/ns-" + i, Set.of("test"));
-        }
     }
 
     @Test
     public void testAutoTopicCreationOverrideWithMaxNumPartitionsLimit() throws Exception{
+        restartClusterAfterTest();
         cleanup();
         conf.setMaxNumPartitionsPerPartitionedTopic(10);
         setup();
@@ -2067,6 +2113,7 @@ public void testAutoTopicCreationOverrideWithMaxNumPartitionsLimit() throws Exce
     }
     @Test
     public void testMaxTopicsPerNamespace() throws Exception {
+        restartClusterAfterTest();
         cleanup();
         conf.setMaxTopicsPerNamespace(10);
         setup();
@@ -2158,16 +2205,12 @@ public void testMaxTopicsPerNamespace() throws Exception {
         } catch (PulsarClientException e) {
             log.info("Exception: ", e);
         }
-
-        // reset configuration
-        conf.setMaxTopicsPerNamespace(0);
-        conf.setDefaultNumPartitions(1);
     }
 
     @Test
     public void testInvalidBundleErrorResponse() throws Exception {
         try {
-            admin.namespaces().deleteNamespaceBundle("prop-xyz/ns1", "invalid-bundle");
+            admin.namespaces().deleteNamespaceBundle(defaultNamespace, "invalid-bundle");
             fail("should have failed due to invalid bundle");
         } catch (PreconditionFailedException e) {
             assertTrue(e.getMessage().startsWith("Invalid bundle range"));
@@ -2176,6 +2219,7 @@ public void testInvalidBundleErrorResponse() throws Exception {
 
     @Test
     public void testMaxSubscriptionsPerTopic() throws Exception {
+        restartClusterAfterTest();
         cleanup();
         conf.setMaxSubscriptionsPerTopic(2);
         setup();
@@ -2258,7 +2302,7 @@ public void testMaxSubscriptionsPerTopic() throws Exception {
 
     @Test(timeOut = 30000)
     public void testMaxSubPerTopicApi() throws Exception {
-        final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+        final String myNamespace = newUniqueName(defaultTenant + "/ns");
         admin.namespaces().createNamespace(myNamespace, Set.of("test"));
 
         assertNull(admin.namespaces().getMaxSubscriptionsPerTopic(myNamespace));
@@ -2281,8 +2325,11 @@ public void testMaxSubPerTopicApi() throws Exception {
         }
     }
 
-    @Test(timeOut = 30000)
+    @Test(timeOut = 60000)
     public void testSetNamespaceEntryFilters() throws Exception {
+        restartClusterAfterTest();
+        restartClusterIfReused();
+        @Cleanup
         final MockEntryFilterProvider testEntryFilterProvider =
                 new MockEntryFilterProvider(conf);
 
@@ -2299,7 +2346,7 @@ public void testSetNamespaceEntryFilters() throws Exception {
         try {
             EntryFilters entryFilters = new EntryFilters("test");
 
-            final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+            final String myNamespace = newUniqueName(defaultTenant + "/ns");
             admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test"));
             final String topicName = myNamespace + "/topic";
             admin.topics().createNonPartitionedTopic(topicName);
@@ -2359,6 +2406,9 @@ public void testSetNamespaceEntryFilters() throws Exception {
 
     @Test(dataProvider = "topicType")
     public void testSetTopicLevelEntryFilters(String topicType) throws Exception {
+        restartClusterAfterTest();
+        restartClusterIfReused();
+        @Cleanup
         final MockEntryFilterProvider testEntryFilterProvider =
                 new MockEntryFilterProvider(conf);
 
@@ -2373,7 +2423,7 @@ public void testSetTopicLevelEntryFilters(String topicType) throws Exception {
                 "entryFilterProvider", testEntryFilterProvider, true);
         try {
             EntryFilters entryFilters = new EntryFilters("test");
-            final String topic = topicType + "://prop-xyz/ns1/test-schema-validation-enforced";
+            final String topic = topicType + "://" + defaultNamespace + "/test-schema-validation-enforced";
             admin.topics().createPartitionedTopic(topic, 1);
             final String fullTopicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + 0;
             @Cleanup
@@ -2431,13 +2481,13 @@ public void testSetTopicLevelEntryFilters(String topicType) throws Exception {
         }
     }
 
-    @Test(timeOut = 30000)
+    @Test(timeOut = 60000)
     public void testSetEntryFiltersHierarchy() throws Exception {
+        restartClusterAfterTest();
+        restartClusterIfReused();
+        @Cleanup
         final MockEntryFilterProvider testEntryFilterProvider =
                 new MockEntryFilterProvider(conf);
-        conf.setEntryFilterNames(List.of("test", "test1"));
-        conf.setAllowOverrideEntryFilters(true);
-
         testEntryFilterProvider.setMockEntryFilters(new EntryFilterDefinition(
                         "test",
                         null,
@@ -2450,9 +2500,11 @@ public void testSetEntryFiltersHierarchy() throws Exception {
         final EntryFilterProvider oldEntryFilterProvider = pulsar.getBrokerService().getEntryFilterProvider();
         FieldUtils.writeField(pulsar.getBrokerService(),
                 "entryFilterProvider", testEntryFilterProvider, true);
+        conf.setEntryFilterNames(List.of("test", "test1"));
+        conf.setAllowOverrideEntryFilters(true);
         try {
 
-            final String topic = "persistent://prop-xyz/ns1/test-schema-validation-enforced";
+            final String topic = "persistent://" + defaultNamespace + "/test-schema-validation-enforced";
             admin.topics().createPartitionedTopic(topic, 1);
             final String fullTopicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + 0;
             @Cleanup
@@ -2460,8 +2512,9 @@ public void testSetEntryFiltersHierarchy() throws Exception {
                     .topic(fullTopicName)
                     .create();
             assertNull(admin.topicPolicies().getEntryFiltersPerTopic(topic, false));
-            assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true),
-                    new EntryFilters("test,test1"));
+            Awaitility.await().untilAsserted(() ->
+                    assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true),
+                            new EntryFilters("test,test1")));
             assertEquals(pulsar
                     .getBrokerService()
                     .getTopic(fullTopicName, false)
@@ -2471,10 +2524,11 @@ public void testSetEntryFiltersHierarchy() throws Exception {
                     .size(), 2);
 
             EntryFilters nsEntryFilters = new EntryFilters("test");
-            admin.namespaces().setNamespaceEntryFilters("prop-xyz/ns1", nsEntryFilters);
-            assertEquals(admin.namespaces().getNamespaceEntryFilters("prop-xyz/ns1"), nsEntryFilters);
-            assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true),
-                    new EntryFilters("test"));
+            admin.namespaces().setNamespaceEntryFilters(defaultNamespace, nsEntryFilters);
+            assertEquals(admin.namespaces().getNamespaceEntryFilters(defaultNamespace), nsEntryFilters);
+            Awaitility.await().untilAsserted(() ->
+                    assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true),
+                            new EntryFilters("test")));
             Awaitility.await().untilAsserted(() -> {
                 assertEquals(pulsar
                         .getBrokerService()
@@ -2503,8 +2557,9 @@ public void testSetEntryFiltersHierarchy() throws Exception {
             admin.topicPolicies().setEntryFiltersPerTopic(topic, topicEntryFilters);
             Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic,
                     false), topicEntryFilters));
-            assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true),
-                    new EntryFilters("test1"));
+            Awaitility.await().untilAsserted(() ->
+                    assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true),
+                            new EntryFilters("test1")));
             Awaitility.await().untilAsserted(() -> {
                 assertEquals(pulsar
                         .getBrokerService()
@@ -2530,8 +2585,11 @@ public void testSetEntryFiltersHierarchy() throws Exception {
         }
     }
 
-    @Test(timeOut = 30000)
+    @Test(timeOut = 60000)
     public void testValidateNamespaceEntryFilters() throws Exception {
+        restartClusterAfterTest();
+        restartClusterIfReused();
+        @Cleanup
         final MockEntryFilterProvider testEntryFilterProvider =
                 new MockEntryFilterProvider(conf);
 
@@ -2546,7 +2604,7 @@ public void testValidateNamespaceEntryFilters() throws Exception {
                 "entryFilterProvider", testEntryFilterProvider, true);
 
         try {
-            final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+            final String myNamespace = newUniqueName(defaultTenant + "/ns");
             admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test"));
             try {
                 admin.namespaces().setNamespaceEntryFilters(myNamespace, new EntryFilters("notexists"));
@@ -2585,8 +2643,11 @@ public void testValidateNamespaceEntryFilters() throws Exception {
         }
     }
 
-    @Test(timeOut = 30000)
+    @Test(timeOut = 60000)
     public void testValidateTopicEntryFilters() throws Exception {
+        restartClusterAfterTest();
+        restartClusterIfReused();
+        @Cleanup
         final MockEntryFilterProvider testEntryFilterProvider =
                 new MockEntryFilterProvider(conf);
 
@@ -2601,7 +2662,7 @@ public void testValidateTopicEntryFilters() throws Exception {
                 "entryFilterProvider", testEntryFilterProvider, true);
 
         try {
-            final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+            final String myNamespace = newUniqueName(defaultTenant + "/ns");
             admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test"));
             final String topicName = myNamespace + "/topic";
             admin.topics().createNonPartitionedTopic(topicName);
@@ -2648,8 +2709,9 @@ public void testValidateTopicEntryFilters() throws Exception {
 
     @Test(timeOut = 30000)
     public void testMaxSubPerTopic() throws Exception {
+        restartClusterAfterTest();
         pulsar.getConfiguration().setMaxSubscriptionsPerTopic(0);
-        final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+        final String myNamespace = newUniqueName(defaultTenant + "/ns");
         admin.namespaces().createNamespace(myNamespace, Set.of("test"));
         final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic";
         pulsarClient.newProducer().topic(topic).create().close();
@@ -2689,12 +2751,13 @@ public void testMaxSubPerTopic() throws Exception {
 
     @Test(timeOut = 30000)
     public void testMaxSubPerTopicPriority() throws Exception {
+        restartClusterAfterTest();
         final int brokerLevelMaxSub = 2;
         cleanup();
         conf.setMaxSubscriptionsPerTopic(brokerLevelMaxSub);
         setup();
 
-        final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+        final String myNamespace = newUniqueName(defaultTenant + "/ns");
         admin.namespaces().createNamespace(myNamespace, Set.of("test"));
         final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic";
         //Create a client that can fail quickly
@@ -2746,12 +2809,13 @@ public void testMaxSubPerTopicPriority() throws Exception {
 
     @Test
     public void testMaxProducersPerTopicUnlimited() throws Exception {
+        restartClusterAfterTest();
         final int maxProducersPerTopic = 1;
         cleanup();
         conf.setMaxProducersPerTopic(maxProducersPerTopic);
         setup();
         //init namespace
-        final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+        final String myNamespace = newUniqueName(defaultTenant + "/ns");
         admin.namespaces().createNamespace(myNamespace, Set.of("test"));
         final String topic = "persistent://" + myNamespace + "/testMaxProducersPerTopicUnlimited";
         //the policy is set to 0, so there will be no restrictions
@@ -2797,12 +2861,13 @@ public void testMaxProducersPerTopicUnlimited() throws Exception {
 
     @Test
     public void testMaxConsumersPerTopicUnlimited() throws Exception {
+        restartClusterAfterTest();
         final int maxConsumersPerTopic = 1;
         cleanup();
         conf.setMaxConsumersPerTopic(maxConsumersPerTopic);
         setup();
         //init namespace
-        final String myNamespace = "prop-xyz/ns" + UUID.randomUUID();
+        final String myNamespace = newUniqueName(defaultTenant + "/ns");
         admin.namespaces().createNamespace(myNamespace, Set.of("test"));
         final String topic = "persistent://" + myNamespace + "/testMaxConsumersPerTopicUnlimited";
 
@@ -2854,7 +2919,7 @@ public void testMaxConsumersPerTopicUnlimited() throws Exception {
 
     @Test
     public void testClearBacklogForTheSubscriptionThatNoConsumers() throws Exception {
-        final String topic = "persistent://prop-xyz/ns1/clear_backlog_no_consumers" + UUID.randomUUID().toString();
+        final String topic = "persistent://" + defaultNamespace + "/clear_backlog_no_consumers" + UUID.randomUUID().toString();
         final String sub = "my-sub";
         admin.topics().createNonPartitionedTopic(topic);
         admin.topics().createSubscription(topic, sub, MessageId.earliest);
@@ -2863,7 +2928,10 @@ public void testClearBacklogForTheSubscriptionThatNoConsumers() throws Exception
 
     @Test(timeOut = 200000)
     public void testCompactionApi() throws Exception {
-        final String namespace = "prop-xyz/ns1";
+        final String namespace = newUniqueName(defaultTenant + "/ns");
+        admin.namespaces().createNamespace(namespace, Set.of("test"));
+
+
         assertNull(admin.namespaces().getCompactionThreshold(namespace));
         assertEquals(pulsar.getConfiguration().getBrokerServiceCompactionThresholdInBytes(), 0);
 
@@ -2878,11 +2946,13 @@ public void testCompactionApi() throws Exception {
 
     @Test(timeOut = 200000)
     public void testCompactionPriority() throws Exception {
+        restartClusterAfterTest();
         cleanup();
         conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10000);
         setup();
-        final String topic = "persistent://prop-xyz/ns1/topic" + UUID.randomUUID();
-        final String namespace = "prop-xyz/ns1";
+        final String namespace = newUniqueName(defaultTenant + "/ns");
+        admin.namespaces().createNamespace(namespace, Set.of("test"));
+        final String topic = "persistent://" + namespace + "/topic" + UUID.randomUUID();
         pulsarClient.newProducer().topic(topic).create().close();
         TopicName topicName = TopicName.get(topic);
         PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get();
@@ -2921,7 +2991,8 @@ public void testCompactionPriority() throws Exception {
 
     @Test
     public void testProperties() throws Exception {
-        final String namespace = "prop-xyz/ns1";
+        final String namespace = newUniqueName(defaultTenant + "/ns");
+        admin.namespaces().createNamespace(namespace, Set.of("test"));
         admin.namespaces().setProperty(namespace, "a", "a");
         assertEquals("a", admin.namespaces().getProperty(namespace, "a"));
         assertNull(admin.namespaces().getProperty(namespace, "b"));
@@ -2944,7 +3015,7 @@ public void testProperties() throws Exception {
 
     @Test
     public void testGetListInBundle() throws Exception {
-        final String namespace = "prop-xyz/ns11";
+        final String namespace = defaultTenant + "/ns11";
         admin.namespaces().createNamespace(namespace, 3);
 
         final String persistentTopicName = TopicName.get(
@@ -2978,7 +3049,8 @@ public void testGetListInBundle() throws Exception {
 
     @Test
     public void testGetTopicsWithDifferentMode() throws Exception {
-        final String namespace = "prop-xyz/ns1";
+        final String namespace = newUniqueName(defaultTenant + "/ns");
+        admin.namespaces().createNamespace(namespace, Set.of("test"));
 
         final String persistentTopicName = TopicName
                 .get("persistent", NamespaceName.get(namespace), "get_topics_mode_" + UUID.randomUUID().toString())
@@ -3021,16 +3093,14 @@ public void testGetTopicsWithDifferentMode() throws Exception {
 
     @Test(dataProvider = "isV1")
     public void testNonPartitionedTopic(boolean isV1) throws Exception {
-        String tenant = "prop-xyz";
+        restartClusterAfterTest();
+        String tenant = defaultTenant;
         String cluster = "test";
         String namespace = tenant + "/" + (isV1 ? cluster + "/" : "") + "n1" + isV1;
         String topic = "persistent://" + namespace + "/t1" + isV1;
         admin.namespaces().createNamespace(namespace, Set.of(cluster));
         admin.topics().createNonPartitionedTopic(topic);
         assertTrue(admin.topics().getList(namespace).contains(topic));
-
-        cleanup();
-        setup();
     }
 
     /**
@@ -3043,7 +3113,7 @@ public void testFailedUpdatePartitionedTopic() throws Exception {
         final String subName1 = topicName + "-my-sub-1";
         final int startPartitions = 4;
         final int newPartitions = 8;
-        final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName;
+        final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + topicName;
 
         URL pulsarUrl = new URL(pulsar.getWebServiceAddress());
 
@@ -3079,11 +3149,9 @@ public void testFailedUpdatePartitionedTopic() throws Exception {
 
     @Test(dataProvider = "topicType")
     public void testPartitionedStatsAggregationByProducerName(String topicType) throws Exception {
-        cleanup();
-        setup();
-
+        restartClusterIfReused();
         conf.setAggregatePublisherStatsByProducerName(true);
-        final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name";
+        final String topic = topicType + "://" + defaultNamespace + "/test-partitioned-stats-aggregation-by-producer-name";
         admin.topics().createPartitionedTopic(topic, 10);
 
         @Cleanup
@@ -3137,8 +3205,9 @@ public int choosePartition(Message msg, TopicMetadata metadata) {
 
     @Test(dataProvider = "topicType")
     public void testPartitionedStatsAggregationByProducerNamePerPartition(String topicType) throws Exception {
+        restartClusterIfReused();
         conf.setAggregatePublisherStatsByProducerName(true);
-        final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name-per-pt";
+        final String topic = topicType + "://" + defaultNamespace + "/test-partitioned-stats-aggregation-by-producer-name-per-pt";
         admin.topics().createPartitionedTopic(topic, 2);
 
         @Cleanup
@@ -3161,7 +3230,7 @@ public void testPartitionedStatsAggregationByProducerNamePerPartition(String top
 
     @Test(dataProvider = "topicType")
     public void testSchemaValidationEnforced(String topicType) throws Exception {
-        final String topic = topicType + "://prop-xyz/ns1/test-schema-validation-enforced";
+        final String topic = topicType + "://" + defaultNamespace + "/test-schema-validation-enforced";
         admin.topics().createPartitionedTopic(topic, 1);
         @Cleanup
         Producer producer1 = pulsarClient.newProducer()
@@ -3177,31 +3246,31 @@ public void testSchemaValidationEnforced(String topicType) throws Exception {
 
     @Test
     public void testGetNamespaceTopicList() throws Exception {
-        final String persistentTopic = "persistent://prop-xyz/ns1/testGetNamespaceTopicList";
-        final String nonPersistentTopic = "non-persistent://prop-xyz/ns1/non-testGetNamespaceTopicList";
-        final String eventTopic = "persistent://prop-xyz/ns1/__change_events";
+        final String persistentTopic = "persistent://" + defaultNamespace + "/testGetNamespaceTopicList";
+        final String nonPersistentTopic = "non-persistent://" + defaultNamespace + "/non-testGetNamespaceTopicList";
+        final String eventTopic = "persistent://" + defaultNamespace + "/__change_events";
         admin.topics().createNonPartitionedTopic(persistentTopic);
         Awaitility.await().untilAsserted(() ->
-                admin.namespaces().getTopics("prop-xyz/ns1",
+                admin.namespaces().getTopics(defaultNamespace,
                 ListNamespaceTopicsOptions.builder().mode(Mode.PERSISTENT).includeSystemTopic(true).build())
                         .contains(eventTopic));
-        List notIncludeSystemTopics = admin.namespaces().getTopics("prop-xyz/ns1",
+        List notIncludeSystemTopics = admin.namespaces().getTopics(defaultNamespace,
                 ListNamespaceTopicsOptions.builder().includeSystemTopic(false).build());
         Assert.assertFalse(notIncludeSystemTopics.contains(eventTopic));
         @Cleanup
         Producer producer = pulsarClient.newProducer()
                 .topic(nonPersistentTopic)
                 .create();
-        List notPersistentTopics = admin.namespaces().getTopics("prop-xyz/ns1",
+        List notPersistentTopics = admin.namespaces().getTopics(defaultNamespace,
                 ListNamespaceTopicsOptions.builder().mode(Mode.NON_PERSISTENT).build());
         Assert.assertTrue(notPersistentTopics.contains(nonPersistentTopic));
     }
 
     @Test
     private void testTerminateSystemTopic() throws Exception {
-        final String topic = "persistent://prop-xyz/ns1/testTerminateSystemTopic";
+        final String topic = "persistent://" + defaultNamespace + "/testTerminateSystemTopic";
         admin.topics().createNonPartitionedTopic(topic);
-        final String eventTopic = "persistent://prop-xyz/ns1/__change_events";
+        final String eventTopic = "persistent://" + defaultNamespace + "/__change_events";
         admin.topicPolicies().setMaxConsumers(topic, 2);
         Awaitility.await().untilAsserted(() -> {
             Assert.assertEquals(admin.topicPolicies().getMaxConsumers(topic), Integer.valueOf(2));
@@ -3213,12 +3282,12 @@ private void testTerminateSystemTopic() throws Exception {
 
     @Test
     private void testDeleteNamespaceForciblyWithManyTopics() throws Exception {
-        final String ns = "prop-xyz/ns-testDeleteNamespaceForciblyWithManyTopics";
+        final String ns = defaultTenant + "/ns-testDeleteNamespaceForciblyWithManyTopics";
         admin.namespaces().createNamespace(ns, 2);
         for (int i = 0; i < 100; i++) {
             admin.topics().createPartitionedTopic(String.format("persistent://%s", ns + "/topic" + i), 3);
         }
         admin.namespaces().deleteNamespace(ns, true);
-        Assert.assertFalse(admin.namespaces().getNamespaces("prop-xyz").contains(ns));
+        Assert.assertFalse(admin.namespaces().getNamespaces(defaultTenant).contains(ns));
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java
index d3d4b7cf9341c..aac4c58457991 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java
@@ -290,6 +290,10 @@ protected void defaultOverrideServiceConfiguration(ServiceConfiguration svcConfi
             if (svcConfig.getManagedLedgerCacheSizeMB() == unconfiguredDefaults.getManagedLedgerCacheSizeMB()) {
                 svcConfig.setManagedLedgerCacheSizeMB(8);
             }
+
+            if (svcConfig.getTopicLoadTimeoutSeconds() == unconfiguredDefaults.getTopicLoadTimeoutSeconds()) {
+                svcConfig.setTopicLoadTimeoutSeconds(10);
+            }
         }
 
         /**

From fe556ab8ac268543004b854dbd7a59e20756dff7 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 7 Jun 2023 22:51:39 +0300
Subject: [PATCH 389/404] [improve][build] Upgrade Testcontainers to 1.18.3 &
 docker-java to 3.3.0 (#20531)

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 7515aaf6863d1..4f14d57faa8f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -251,11 +251,11 @@ flexible messaging model and an intuitive client API.
     2.0.6
 
     
-    1.17.6
+    1.18.3
     2.2
 
     
-    3.2.13
+    3.3.0
     1.1.1
     7.7.1
     3.12.4

From ac46e2e4fc48dff74233623afa3635ef5285e34d Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Wed, 7 Jun 2023 22:52:13 +0300
Subject: [PATCH 390/404] [fix][broker] Disable EntryFilters for system topics
 (#20514)

---
 .../apache/pulsar/broker/service/AbstractTopic.java |  4 ++++
 .../pulsar/broker/service/EntryFilterSupport.java   |  3 ++-
 .../broker/service/persistent/SystemTopic.java      | 13 +++++++++++++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 4614b846c8eee..1371019be41dc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -1163,6 +1163,10 @@ public void updateResourceGroupLimiter(Optional optPolicies) {
     }
 
     public void updateEntryFilters() {
+        if (isSystemTopic()) {
+            entryFilters = Pair.of(null, Collections.emptyList());
+            return;
+        }
         final EntryFilters entryFiltersPolicy = getEntryFiltersPolicy();
         if (entryFiltersPolicy == null || StringUtils.isBlank(entryFiltersPolicy.getEntryFilterNames())) {
             entryFilters = Pair.of(null, Collections.emptyList());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java
index 4a9b33a9afdb1..03d6f0750e02e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java
@@ -35,7 +35,8 @@ public class EntryFilterSupport {
 
     public EntryFilterSupport(Subscription subscription) {
         this.subscription = subscription;
-        if (subscription != null && subscription.getTopic() != null) {
+        if (subscription != null && subscription.getTopic() != null
+                && !subscription.getTopic().isSystemTopic()) {
             final BrokerService brokerService = subscription.getTopic().getBrokerService();
             final boolean allowOverrideEntryFilters = brokerService
                     .pulsar().getConfiguration().isAllowOverrideEntryFilters();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java
index 395a8c9075eb3..720ae3c51891e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java
@@ -18,13 +18,16 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.service.plugin.EntryFilter;
 import org.apache.pulsar.common.naming.SystemTopicNames;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.EntryFilters;
 
 public class SystemTopic extends PersistentTopic {
 
@@ -82,4 +85,14 @@ public boolean isEncryptionRequired() {
         // System topics are only written by the broker that can't know the encryption context.
         return false;
     }
+
+    @Override
+    public EntryFilters getEntryFiltersPolicy() {
+        return null;
+    }
+
+    @Override
+    public List getEntryFilters() {
+        return null;
+    }
 }

From 19face6dc9292ff40d710909b4444b01b15a2378 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Thu, 8 Jun 2023 13:58:08 +0800
Subject: [PATCH 391/404] [fix][broker] Fix redirect loop when using
 ExtensibleLoadManager and list in bundle admin API (#20528)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
When using `ExtensibleLoadManager` and list in bundle admin API,
it will redirect forever because `isServiceUnitOwned` method is checking the `ownershipCache` as the ownership storage, however, when using `ExtensibleLoadManager`, it stored the ownership to table view.

### Modifications

* Call `isServiceUnitOwnedAsync ` when using `isServiceUnitOwned `.
* Add unit test to cover this case.
---
 .../broker/namespace/NamespaceService.java    | 18 +--------
 .../ExtensibleLoadManagerImplTest.java        | 38 +++++++++++++++++++
 2 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index cf969460c3345..3997ab521b889 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1108,19 +1108,7 @@ public Set getOwnedServiceUnits() {
     }
 
     public boolean isServiceUnitOwned(ServiceUnitId suName) throws Exception {
-        if (suName instanceof TopicName) {
-            return isTopicOwnedAsync((TopicName) suName).get();
-        }
-
-        if (suName instanceof NamespaceName) {
-            return isNamespaceOwned((NamespaceName) suName);
-        }
-
-        if (suName instanceof NamespaceBundle) {
-            return ownershipCache.isNamespaceBundleOwned((NamespaceBundle) suName);
-        }
-
-        throw new IllegalArgumentException("Invalid class of NamespaceBundle: " + suName.getClass().getName());
+        return isServiceUnitOwnedAsync(suName).get(config.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
     }
 
     public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) {
@@ -1174,10 +1162,6 @@ public CompletableFuture isServiceUnitActiveAsync(TopicName topicName)
         });
     }
 
-    private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception {
-        return ownershipCache.getOwnedBundle(getFullBundle(fqnn)) != null;
-    }
-
     private CompletableFuture isNamespaceOwnedAsync(NamespaceName fqnn) {
         // TODO: Add unit tests cover it.
         if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index e8a4682e528d5..d1c8ddbb4d3e9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -55,6 +55,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -926,6 +927,43 @@ public void testDisableBroker() throws Exception {
         }
     }
 
+    @Test(timeOut = 30 * 1000)
+    public void testListTopic() throws Exception {
+        final String namespace = "public/testListTopic";
+        admin.namespaces().createNamespace(namespace, 3);
+
+        final String persistentTopicName = TopicName.get(
+                "persistent", NamespaceName.get(namespace),
+                "get_topics_mode_" + UUID.randomUUID()).toString();
+
+        final String nonPersistentTopicName = TopicName.get(
+                "non-persistent", NamespaceName.get(namespace),
+                "get_topics_mode_" + UUID.randomUUID()).toString();
+        admin.topics().createPartitionedTopic(persistentTopicName, 3);
+        admin.topics().createPartitionedTopic(nonPersistentTopicName, 3);
+        pulsarClient.newProducer().topic(persistentTopicName).create().close();
+        pulsarClient.newProducer().topic(nonPersistentTopicName).create().close();
+
+        BundlesData bundlesData = admin.namespaces().getBundles(namespace);
+        List boundaries = bundlesData.getBoundaries();
+        int topicNum = 0;
+        for (int i = 0; i < boundaries.size() - 1; i++) {
+            String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1));
+            List topic = admin.topics().getListInBundle(namespace, bundle);
+            if (topic == null) {
+                continue;
+            }
+            topicNum += topic.size();
+            for (String s : topic) {
+                assertFalse(TopicName.get(s).isPersistent());
+            }
+        }
+        assertEquals(topicNum, 3);
+
+        List list = admin.topics().getList(namespace);
+        assertEquals(list.size(), 6);
+    }
+
     private static abstract class MockBrokerFilter implements BrokerFilter {
 
         @Override

From c39f7bb123a995538fc8fa9a54b572c73fee39db Mon Sep 17 00:00:00 2001
From: vineeth1995 
Date: Thu, 8 Jun 2023 01:10:27 -0700
Subject: [PATCH 392/404] [improve][admin] Pretty print bookies racks-placement
 command output (#20516)

---
 .../java/org/apache/pulsar/admin/cli/CliCommand.java   | 10 +++++++++-
 .../java/org/apache/pulsar/admin/cli/CmdBookies.java   |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java
index 7a6836eb74762..c96b0bd4365d2 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java
@@ -214,13 +214,21 @@  void print(T item) {
             if (item instanceof String) {
                 System.out.println(item);
             } else {
-                System.out.println(writer.writeValueAsString(item));
+                prettyPrint(item);
             }
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
+     void prettyPrint(T item) {
+        try {
+            System.out.println(writer.writeValueAsString(item));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private static ObjectMapper mapper = ObjectMapperFactory.create();
     private static ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
     private static Set sizeUnit = Sets.newHashSet('k', 'K', 'm', 'M', 'g', 'G', 't', 'T');
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java
index 8c8f0f4e8a2d1..27502a305ac4d 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java
@@ -35,7 +35,7 @@ private class GetAll extends CliCommand {
 
         @Override
         void run() throws Exception {
-            print(getAdmin().bookies().getBookiesRackInfo());
+            prettyPrint(getAdmin().bookies().getBookiesRackInfo());
         }
     }
 

From 362ece39df6c1788f45e7cc52c4f09e12a0ff515 Mon Sep 17 00:00:00 2001
From: Ruguo Yu 
Date: Thu, 8 Jun 2023 23:20:46 +0800
Subject: [PATCH 393/404] [fix][admin] Make admin.cli throw exception instead
 of silent catch (#20530)

---
 .../main/java/org/apache/pulsar/admin/cli/CmdClusters.java    | 4 +---
 .../main/java/org/apache/pulsar/admin/cli/CmdFunctions.java   | 4 +---
 .../src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java   | 4 +---
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java
index 173595c9b19a4..6d6b4e72268cb 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java
@@ -268,11 +268,9 @@ void run() throws Exception {
             try {
                 processArguments();
             } catch (Exception e) {
-                System.err.println(e.getMessage());
-                System.err.println();
                 String chosenCommand = jcommander.getParsedCommand();
                 getUsageFormatter().usage(chosenCommand);
-                return;
+                throw e;
             }
             runCmd();
         }
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 9b30d59f1679c..c94041df0ffaf 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -91,11 +91,9 @@ void run() throws Exception {
             try {
                 processArguments();
             } catch (Exception e) {
-                System.err.println(e.getMessage());
-                System.err.println();
                 String chosenCommand = jcommander.getParsedCommand();
                 getUsageFormatter().usage(chosenCommand);
-                return;
+                throw e;
             }
             runCmd();
         }
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
index 0b27dd8d0a737..6d9619244a32e 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
@@ -115,11 +115,9 @@ void run() throws Exception {
             try {
                 processArguments();
             } catch (Exception e) {
-                System.err.println(e.getMessage());
-                System.err.println();
                 String chosenCommand = jcommander.getParsedCommand();
                 getUsageFormatter().usage(chosenCommand);
-                return;
+                throw e;
             }
             runCmd();
         }

From aa1d5d9ab660803179ee2f24dcedd23428ec867c Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Fri, 9 Jun 2023 08:35:46 +0800
Subject: [PATCH 394/404] [fix][broker] Should not throw NotFoundException when
 metadata already exists. (#20539)

## Motivation
Make the Exception more clear.
## Modification
Return the `409 Conflict` instead of `404 NoFound` when metadata already exists.
---
 .../pulsar/broker/admin/impl/PackagesBase.java  |  3 +++
 .../pulsar/broker/admin/v3/PackagesApiTest.java | 17 +++++++++++++++++
 .../core/impl/PackagesManagementImpl.java       |  5 +++--
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java
index c4f1db3205038..615cc6cb7a6ac 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.admin.impl;
 
 import java.io.InputStream;
+import java.nio.file.FileAlreadyExistsException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import javax.ws.rs.WebApplicationException;
@@ -64,6 +65,8 @@ private Void handleError(Throwable throwable, AsyncResponse asyncResponse) {
             asyncResponse.resume(throwable);
         } else if (throwable instanceof UnsupportedOperationException) {
             asyncResponse.resume(new RestException(Response.Status.SERVICE_UNAVAILABLE, throwable.getMessage()));
+        } else if (throwable instanceof FileAlreadyExistsException) {
+            asyncResponse.resume(new RestException(Response.Status.CONFLICT, throwable.getMessage()));
         } else {
             log.error("Encountered unexpected error", throwable);
             asyncResponse.resume(new RestException(Response.Status.INTERNAL_SERVER_ERROR, throwable.getMessage()));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java
index 7085ed178fc28..221e2d47bfa1a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java
@@ -56,6 +56,23 @@ protected void cleanup() throws Exception {
         super.internalCleanup();
     }
 
+    @Test
+    public void testRepeatUploadThrowConflictException() throws Exception {
+        // create a temp file for testing
+        File file = File.createTempFile("package-api-test", ".package");
+
+        // testing upload api
+        String packageName = "function://public/default/test@v1";
+        PackageMetadata originalMetadata = PackageMetadata.builder().description("test").build();
+        admin.packages().upload(originalMetadata, packageName, file.getPath());
+        try {
+            admin.packages().upload(originalMetadata, packageName, file.getPath());
+            fail();
+        } catch (PulsarAdminException e) {
+            assertEquals(e.getStatusCode(), 409);
+        }
+    }
+
     @Test(timeOut = 60000)
     public void testPackagesOperations() throws Exception {
         // create a temp file for testing
diff --git a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java
index 449d1268f3fd5..7861d07c775ce 100644
--- a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java
+++ b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java
@@ -23,6 +23,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.file.FileAlreadyExistsException;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import org.apache.pulsar.packages.management.core.PackagesManagement;
@@ -187,8 +188,8 @@ private CompletableFuture checkMetadataExistsAndThrowException(PackageName
                 if (!exist) {
                     future.complete(null);
                 } else {
-                    future.completeExceptionally(
-                        new NotFoundException(String.format("Package '%s' metadata already exists", packageName)));
+                    future.completeExceptionally(new FileAlreadyExistsException(
+                            String.format("Package '%s' metadata already exists", packageName)));
                 }
             });
         return future;

From 005cce11d0d67cdf7b08305bed55cd2f9ba7f4f0 Mon Sep 17 00:00:00 2001
From: maanders-tibco <84395784+maanders-tibco@users.noreply.github.com>
Date: Thu, 8 Jun 2023 21:29:00 -0500
Subject: [PATCH 395/404] [fix][broker] REST Client Producer fails with TLS
 only (#20535)

Co-authored-by: Matt Anderson <>

Fixes #20536

### Motivation

When disabling HTTP ports in the Pulsar broker, the [REST Client Producer](https://pulsar.apache.org/docs/3.0.x/client-libraries-rest/) fails to produce messages. For reproduction steps, please reference issue details.

### Modifications

Change the following lines:
```java

            LookupResult result = optionalResult.get();
            if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())) {
                // Current broker owns the topic, add to owning topic.
```
To:
```java
            LookupResult result = optionalResult.get();
            if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())
                    || result.getLookupData().getHttpUrlTls().equals(pulsar().getWebServiceAddressTls())) {
                // Current broker owns the topic, add to owning topic.
```

### Verifying this change

This change is a trivial rework / code cleanup without any test coverage. (outside of the reproduction tests described in the #20536)



*If the box was checked, please highlight the changes*

- [ ] Dependencies (add or upgrade a dependency)
- [ ] The public API
- [ ] The schema
- [ ] The default values of configurations
- [ ] The threading model
- [ ] The binary protocol
- [x] The REST endpoints
- [ ] The admin CLI options
- [ ] The metrics
- [ ] Anything that affects deployment

### Documentation



- [ ] `doc` 
- [ ] `doc-required` 
- [x] `doc-not-needed` 
- [ ] `doc-complete` 

### Matching PR in forked repository

PR in forked repository: https://github.com/maanders-tibco/pulsar


---
 .../main/java/org/apache/pulsar/broker/rest/TopicsBase.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
index 6f3ac7f8c09ca..5ab81fcbbff27 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java
@@ -433,7 +433,8 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN
             }
 
             LookupResult result = optionalResult.get();
-            if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())) {
+            if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())
+                    || result.getLookupData().getHttpUrlTls().equals(pulsar().getWebServiceAddressTls())) {
                 // Current broker owns the topic, add to owning topic.
                 if (log.isDebugEnabled()) {
                     log.debug("Complete topic look up for rest produce message request for topic {}, "

From d0d0338a18d853c1bcbe15d72bdaf57b4eedd7dd Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Fri, 9 Jun 2023 10:40:27 +0800
Subject: [PATCH 396/404] [improve][broker] Emit the namespace bundle listener
 event on extensible load manager (#20525)

---
 .../channel/ServiceUnitStateChannelImpl.java  |   3 +
 .../broker/namespace/NamespaceService.java    |   6 +-
 .../ExtensibleLoadManagerImplTest.java        | 103 +++++++++++++++---
 3 files changed, 93 insertions(+), 19 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 489a00851057b..48192b5dc7cd8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -703,6 +703,7 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) {
         stateChangeListeners.notify(serviceUnit, data, null);
         if (isTargetBroker(data.dstBroker())) {
             log(null, serviceUnit, data, null);
+            pulsar.getNamespaceService().onNamespaceBundleOwned(getNamespaceBundle(serviceUnit));
             lastOwnEventHandledAt = System.currentTimeMillis();
         } else if (data.force() && isTargetBroker(data.sourceBroker())) {
             closeServiceUnit(serviceUnit);
@@ -841,6 +842,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit) {
                 .whenComplete((__, ex) -> {
                     // clean up topics that failed to unload from the broker ownership cache
                     pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle);
+                    pulsar.getNamespaceService().onNamespaceBundleUnload(bundle);
                     double unloadBundleTime = TimeUnit.NANOSECONDS
                             .toMillis((System.nanoTime() - startTime));
                     if (ex != null) {
@@ -912,6 +914,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                     double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime));
                     log.info("Successfully split {} parent namespace-bundle to {} in {} ms",
                             parentBundle, childBundles, splitBundleTime);
+                    namespaceService.onNamespaceBundleSplit(parentBundle);
                     completionFuture.complete(null);
                 })
                 .exceptionally(ex -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 3997ab521b889..69c25d7c6d394 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1202,13 +1202,13 @@ public CompletableFuture removeOwnedServiceUnitAsync(NamespaceBundle nsBun
         return future.thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject()));
     }
 
-    protected void onNamespaceBundleOwned(NamespaceBundle bundle) {
+    public void onNamespaceBundleOwned(NamespaceBundle bundle) {
         for (NamespaceBundleOwnershipListener bundleOwnedListener : bundleOwnershipListeners) {
             notifyNamespaceBundleOwnershipListener(bundle, bundleOwnedListener);
         }
     }
 
-    protected void onNamespaceBundleUnload(NamespaceBundle bundle) {
+    public void onNamespaceBundleUnload(NamespaceBundle bundle) {
         for (NamespaceBundleOwnershipListener bundleOwnedListener : bundleOwnershipListeners) {
             try {
                 if (bundleOwnedListener.test(bundle)) {
@@ -1220,7 +1220,7 @@ protected void onNamespaceBundleUnload(NamespaceBundle bundle) {
         }
     }
 
-    protected void onNamespaceBundleSplit(NamespaceBundle bundle) {
+    public void onNamespaceBundleSplit(NamespaceBundle bundle) {
         for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) {
             try {
                 if (bundleSplitListener.test(bundle)) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index d1c8ddbb4d3e9..1181c18ce40fe 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -87,6 +87,8 @@
 import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.namespace.LookupOptions;
+import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener;
+import org.apache.pulsar.broker.namespace.NamespaceBundleSplitListener;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.impl.TableViewImpl;
@@ -126,6 +128,8 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest {
     private ServiceUnitStateChannelImpl channel1;
     private ServiceUnitStateChannelImpl channel2;
 
+    private final String defaultTestNamespace = "public/test";
+
     @BeforeClass
     @Override
     public void setup() throws Exception {
@@ -136,6 +140,7 @@ public void setup() throws Exception {
         conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
         conf.setLoadBalancerSheddingEnabled(false);
         conf.setLoadBalancerDebugModeEnabled(true);
+        conf.setTopicLevelPoliciesEnabled(false);
         super.internalSetup(conf);
         pulsar1 = pulsar;
         ServiceConfiguration defaultConf = getDefaultConf();
@@ -144,6 +149,7 @@ public void setup() throws Exception {
         defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
         defaultConf.setLoadBalancerSheddingEnabled(false);
+        defaultConf.setTopicLevelPoliciesEnabled(false);
         additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf);
         pulsar2 = additionalPulsarTestContext.getPulsarService();
 
@@ -159,6 +165,10 @@ public void setup() throws Exception {
         admin.namespaces().createNamespace("public/default");
         admin.namespaces().setNamespaceReplicationClusters("public/default",
                 Sets.newHashSet(this.conf.getClusterName()));
+
+        admin.namespaces().createNamespace(defaultTestNamespace);
+        admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace,
+                Sets.newHashSet(this.conf.getClusterName()));
     }
 
     @Override
@@ -172,7 +182,7 @@ protected void cleanup() throws Exception {
 
     @BeforeMethod(alwaysRun = true)
     protected void initializeState() throws PulsarAdminException {
-        admin.namespaces().unload("public/default");
+        admin.namespaces().unload(defaultTestNamespace);
         reset(primaryLoadManager, secondaryLoadManager);
     }
 
@@ -196,7 +206,7 @@ public void testAssignInternalTopic() throws Exception {
 
     @Test
     public void testAssign() throws Exception {
-        TopicName topicName = TopicName.get("test-assign");
+        TopicName topicName = TopicName.get(defaultTestNamespace + "/test-assign");
         NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
         Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get();
         assertTrue(brokerLookupData.isPresent());
@@ -221,7 +231,8 @@ public void testAssign() throws Exception {
 
     @Test
     public void testCheckOwnershipAsync() throws Exception {
-        NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test-check-ownership")).get();
+        TopicName topicName = TopicName.get(defaultTestNamespace + "/test-check-ownership");
+        NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
         // 1. The bundle is never assigned.
         assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
         assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
@@ -241,7 +252,7 @@ public void testCheckOwnershipAsync() throws Exception {
 
     @Test
     public void testFilter() throws Exception {
-        TopicName topicName = TopicName.get("test-filter");
+        TopicName topicName = TopicName.get(defaultTestNamespace + "/test-filter");
         NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
 
         doReturn(List.of(new BrokerFilter() {
@@ -267,7 +278,7 @@ public Map filter(Map broker
 
     @Test
     public void testFilterHasException() throws Exception {
-        TopicName topicName = TopicName.get("test-filter-has-exception");
+        TopicName topicName = TopicName.get(defaultTestNamespace + "/test-filter-has-exception");
         NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
 
         doReturn(List.of(new MockBrokerFilter() {
@@ -286,20 +297,58 @@ public Map filter(Map broker
 
     @Test(timeOut = 30 * 1000)
     public void testUnloadAdminAPI() throws Exception {
-        TopicName topicName = TopicName.get("test-unload");
+        TopicName topicName = TopicName.get(defaultTestNamespace + "/test-unload");
         NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
 
+        AtomicInteger onloadCount = new AtomicInteger(0);
+        AtomicInteger unloadCount = new AtomicInteger(0);
+
+        NamespaceBundleOwnershipListener listener = new NamespaceBundleOwnershipListener() {
+            @Override
+            public void onLoad(NamespaceBundle bundle) {
+                onloadCount.incrementAndGet();
+            }
+
+            @Override
+            public void unLoad(NamespaceBundle bundle) {
+                unloadCount.incrementAndGet();
+            }
+
+            @Override
+            public boolean test(NamespaceBundle namespaceBundle) {
+                return namespaceBundle.equals(bundle);
+            }
+        };
+        pulsar1.getNamespaceService().addNamespaceBundleOwnershipListener(listener);
+        pulsar2.getNamespaceService().addNamespaceBundleOwnershipListener(listener);
         String broker = admin.lookups().lookupTopic(topicName.toString());
         log.info("Assign the bundle {} to {}", bundle, broker);
 
         checkOwnershipState(broker, bundle);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(onloadCount.get(), 1);
+            assertEquals(unloadCount.get(), 0);
+        });
+
         admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange());
         assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
         assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(onloadCount.get(), 1);
+            assertEquals(unloadCount.get(), 1);
+        });
 
         broker = admin.lookups().lookupTopic(topicName.toString());
         log.info("Assign the bundle {} to {}", bundle, broker);
 
+        String finalBroker = broker;
+        Awaitility.await().untilAsserted(() -> {
+            checkOwnershipState(finalBroker, bundle);
+            assertEquals(onloadCount.get(), 2);
+            assertEquals(unloadCount.get(), 1);
+        });
+
+
         String dstBrokerUrl = pulsar1.getLookupServiceAddress();
         String dstBrokerServiceUrl;
         if (broker.equals(pulsar1.getBrokerServiceUrl())) {
@@ -311,6 +360,10 @@ public void testUnloadAdminAPI() throws Exception {
         checkOwnershipState(broker, bundle);
 
         admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(onloadCount.get(), 3);
+            assertEquals(unloadCount.get(), 2);
+        });
 
         assertEquals(admin.lookups().lookupTopic(topicName.toString()), dstBrokerServiceUrl);
 
@@ -338,7 +391,7 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle)
 
     @Test(timeOut = 30 * 1000)
     public void testSplitBundleAdminAPI() throws Exception {
-        String namespace = "public/default";
+        String namespace = defaultTestNamespace;
         String topic = "persistent://" + namespace + "/test-split";
         admin.topics().createPartitionedTopic(topic, 10);
         BundlesData bundles = admin.namespaces().getBundles(namespace);
@@ -347,6 +400,23 @@ public void testSplitBundleAdminAPI() throws Exception {
 
         String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1);
 
+        AtomicInteger splitCount = new AtomicInteger(0);
+        NamespaceBundleSplitListener namespaceBundleSplitListener = new NamespaceBundleSplitListener() {
+            @Override
+            public void onSplit(NamespaceBundle bundle) {
+                splitCount.incrementAndGet();
+            }
+
+            @Override
+            public boolean test(NamespaceBundle namespaceBundle) {
+                return namespaceBundle
+                        .toString()
+                        .equals(String.format(namespace + "/0x%08x_0x%08x", bundleRanges.get(0), bundleRanges.get(1)));
+            }
+        };
+        pulsar1.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener);
+        pulsar2.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener);
+
         long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2;
 
         admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null);
@@ -359,6 +429,7 @@ public void testSplitBundleAdminAPI() throws Exception {
         assertTrue(bundlesData.getBoundaries().contains(lowBundle));
         assertTrue(bundlesData.getBoundaries().contains(midBundle));
         assertTrue(bundlesData.getBoundaries().contains(highBundle));
+        assertEquals(splitCount.get(), 1);
 
         // Test split bundle with invalid bundle range.
         try {
@@ -371,7 +442,7 @@ public void testSplitBundleAdminAPI() throws Exception {
 
     @Test(timeOut = 30 * 1000)
     public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception {
-        String namespace = "public/default";
+        String namespace = defaultTestNamespace;
         String topic = "persistent://" + namespace + "/test-split-with-specific-position";
         admin.topics().createPartitionedTopic(topic, 10);
         BundlesData bundles = admin.namespaces().getBundles(namespace);
@@ -398,7 +469,9 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception {
     }
     @Test(timeOut = 30 * 1000)
     public void testDeleteNamespaceBundle() throws Exception {
-        TopicName topicName = TopicName.get("test-delete-namespace-bundle");
+        final String namespace = "public/testDeleteNamespaceBundle";
+        admin.namespaces().createNamespace(namespace, 3);
+        TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-bundle");
         NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
 
         String broker = admin.lookups().lookupTopic(topicName.toString());
@@ -447,7 +520,7 @@ public void testCheckOwnershipPresentWithSystemNamespace() throws Exception {
 
     @Test
     public void testMoreThenOneFilter() throws Exception {
-        TopicName topicName = TopicName.get("test-filter-has-exception");
+        TopicName topicName = TopicName.get(defaultTestNamespace + "/test-filter-has-exception");
         NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
 
         String lookupServiceAddress1 = pulsar1.getLookupServiceAddress();
@@ -485,7 +558,7 @@ public void testDeployAndRollbackLoadManager() throws Exception {
         try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) {
             // start pulsar3 with old load manager
             var pulsar3 = additionalPulsarTestContext.getPulsarService();
-            String topic = "persistent://public/default/test";
+            String topic = "persistent://" + defaultTestNamespace + "/test";
 
             String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic);
             assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl());
@@ -594,7 +667,7 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except
         restartBroker();
         pulsar1 = pulsar;
         setPrimaryLoadManager();
-        admin.namespaces().setNamespaceReplicationClusters("public/default",
+        admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace,
                 Sets.newHashSet(this.conf.getClusterName()));
 
         var serviceUnitStateChannelPrimaryNew =
@@ -614,10 +687,7 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except
     }
 
     @Test
-    public void testRoleChange()
-            throws Exception {
-
-
+    public void testRoleChange() throws Exception {
         var topBundlesLoadDataStorePrimary = (LoadDataStore)
                 FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true);
         var topBundlesLoadDataStorePrimarySpy = spy(topBundlesLoadDataStorePrimary);
@@ -962,6 +1032,7 @@ public void testListTopic() throws Exception {
 
         List list = admin.topics().getList(namespace);
         assertEquals(list.size(), 6);
+        admin.namespaces().deleteNamespace(namespace, true);
     }
 
     private static abstract class MockBrokerFilter implements BrokerFilter {

From 05f7e62cc0ebe50f212559f0aac19d946eb1478c Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 9 Jun 2023 11:42:29 +0300
Subject: [PATCH 397/404] [fix][build] Configure git-commit-id-plugin to skip
 git describe (#20550)

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4f14d57faa8f6..1c7cbb5de5516 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1588,10 +1588,10 @@ flexible messaging model and an intuitive client API.
           true
           git
           false
+          false
           false
           
-            false
-            false
+            true
           
         
       

From 45d882a2d864398f55fb786c1ef20cd42b9e99b4 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Fri, 9 Jun 2023 17:22:26 +0800
Subject: [PATCH 398/404] [improve][broker] Save createIfMissing in
 TopicLoadingContext (#19993)

---
 .../pulsar/broker/service/BrokerService.java  |  9 ++-
 .../broker/service/BrokerServiceTest.java     | 63 +++++++++++++++++++
 2 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index c38514237856b..39635fa673554 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1533,7 +1533,8 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S
                             return null;
                         });
                     } else {
-                        pendingTopicLoadingQueue.add(new TopicLoadingContext(topic, topicFuture, properties));
+                        pendingTopicLoadingQueue.add(new TopicLoadingContext(topic,
+                                createIfMissing, topicFuture, properties));
                         if (log.isDebugEnabled()) {
                             log.debug("topic-loading for {} added into pending queue", topic);
                         }
@@ -3015,7 +3016,10 @@ private void createPendingLoadTopic() {
             CompletableFuture> pendingFuture = pendingTopic.getTopicFuture();
             final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get();
             final boolean acquiredPermit = topicLoadSemaphore.tryAcquire();
-            checkOwnershipAndCreatePersistentTopic(topic, true, pendingFuture, pendingTopic.getProperties());
+            checkOwnershipAndCreatePersistentTopic(topic,
+                    pendingTopic.isCreateIfMissing(),
+                    pendingFuture,
+                    pendingTopic.getProperties());
             pendingFuture.handle((persistentTopic, ex) -> {
                 // release permit and process next pending topic
                 if (acquiredPermit) {
@@ -3513,6 +3517,7 @@ public void setPulsarChannelInitializerFactory(PulsarChannelInitializer.Factory
     @Getter
     private static class TopicLoadingContext {
         private final String topic;
+        private final boolean createIfMissing;
         private final CompletableFuture> topicFuture;
         private final Map properties;
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index d815445b5fa65..d0c128eb89942 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -41,6 +41,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -1154,6 +1155,68 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception {
         }
     }
 
+    @Test
+    public void testConcurrentLoadTopicExceedLimitShouldNotBeAutoCreated() throws Exception {
+        boolean needDeleteTopic = false;
+        final String namespace = "prop/concurrentLoad";
+        try {
+            // set up broker disable auto create and set concurrent load to 1 qps.
+            cleanup();
+            conf.setMaxConcurrentTopicLoadRequest(1);
+            conf.setAllowAutoTopicCreation(false);
+            setup();
+
+            try {
+                admin.namespaces().createNamespace(namespace);
+            } catch (PulsarAdminException.ConflictException e) {
+                // Ok.. (if test fails intermittently and namespace is already created)
+            }
+
+            // create 3 topic
+            String topicName = "persistent://" + namespace + "/my-topic";
+
+            for (int i = 0; i < 3; i++) {
+                admin.topics().createNonPartitionedTopic(topicName + "_" + i);
+            }
+
+            needDeleteTopic = true;
+
+            // try to load 10 topic
+            ArrayList>> loadFutures = new ArrayList<>();
+            for (int i = 0; i < 10; i++) {
+                // try to create topic which should fail as bundle is disable
+                CompletableFuture> futureResult = pulsar.getBrokerService()
+                        .loadOrCreatePersistentTopic(topicName + "_" + i, false, null);
+                loadFutures.add(futureResult);
+            }
+
+            CompletableFuture[] o = (CompletableFuture[]) Array.newInstance(CompletableFuture.class, 10);
+            CompletableFuture[] completableFutures = loadFutures.toArray(o);
+            CompletableFuture.allOf(completableFutures).get();
+
+            // check topic load CompletableFuture. only first three topic should be success.
+            for (int i = 0; i < 10; i++) {
+                CompletableFuture> load = loadFutures.get(i);
+                if (i < 3) {
+                    Assert.assertTrue(load.isDone());
+                    Assert.assertFalse(load.isCompletedExceptionally());
+                } else {
+                    // check topic should not be created if disable autoCreateTopic.
+                    Assert.assertTrue(load.isDone());
+                    Assert.assertTrue(load.get().isEmpty());
+                }
+            }
+        } finally {
+            if (needDeleteTopic) {
+                String topicName = "persistent://" + namespace + "/my-topic";
+
+                for (int i = 0; i < 3; i++) {
+                    admin.topics().delete(topicName + "_" + i);
+                }
+            }
+        }
+    }
+
     /**
      * Verifies brokerService should not have deadlock and successfully remove topic from topicMap on topic-failure and
      * it should not introduce deadlock while performing it.

From c73967c811f60d4cb508e8489e6faf39dd0174b4 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Fri, 9 Jun 2023 16:39:23 -0500
Subject: [PATCH 399/404] [cleanup][broker] Validate authz earlier in delete
 subscription logic (#20549)

### Motivation

Move the authorization check a few steps earlier in the delete subscription admin endpoint.

### Modifications

* Move the authz check earlier

### Verifying this change

We do not have any tests for these endpoints. We should add them. This change is trivial enough that I think it is fine to defer on testing the authz change.

### Documentation

- [x] `doc-not-needed`
---
 .../pulsar/broker/admin/impl/PersistentTopicsBase.java      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 03421b820dafb..952a775c4af5d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -1597,7 +1597,9 @@ protected CompletableFuture internalDeleteSubscriptionAsync(String subName
             future = CompletableFuture.completedFuture(null);
         }
 
-        return future.thenCompose(__ -> {
+        return future
+                .thenCompose((__) -> validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName))
+                .thenCompose(__ -> {
             if (topicName.isPartitioned()) {
                 return internalDeleteSubscriptionForNonPartitionedTopicAsync(subName, authoritative, force);
             } else {
@@ -1640,11 +1642,11 @@ protected CompletableFuture internalDeleteSubscriptionAsync(String subName
         });
     }
 
+    // Note: this method expects the caller to check authorization
     private CompletableFuture internalDeleteSubscriptionForNonPartitionedTopicAsync(String subName,
                                                                                           boolean authoritative,
                                                                                           boolean force) {
         return validateTopicOwnershipAsync(topicName, authoritative)
-                .thenCompose((__) -> validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName))
                 .thenCompose(__ -> getTopicReferenceAsync(topicName))
                 .thenCompose((topic) -> {
                     Subscription sub = topic.getSubscription(subName);

From 791282efe920c7b709b019f0decc7e647b9da052 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Sun, 11 Jun 2023 18:19:01 +0800
Subject: [PATCH 400/404] [fix][misc] Fix typo in  pip template  (#20556)

---
 pip/TEMPLATE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pip/TEMPLATE.md b/pip/TEMPLATE.md
index 6f907eef7e8e9..549916c7eb929 100644
--- a/pip/TEMPLATE.md
+++ b/pip/TEMPLATE.md
@@ -132,7 +132,7 @@ An important aspect to consider is also multi-tenancy: Does the feature I'm addi
 If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list.
 -->
 
-# Backward & Forward Compatability
+# Backward & Forward Compatibility
 
 ## Revert
 

From 8ffa2dfde29a69ca3a86f831eaae29c95001e735 Mon Sep 17 00:00:00 2001
From: Zhang Yuxuan <61108539+zuobiao-zhou@users.noreply.github.com>
Date: Mon, 12 Jun 2023 12:47:22 +0800
Subject: [PATCH 401/404] [fix][doc] change the method of generating reference
 docs titles. (#20557)

---
 .../java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java   | 2 +-
 .../java/org/apache/pulsar/common/util/CmdGenerateDocs.java     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java
index 764351a4f5e2e..cab037faf8ffc 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java
@@ -108,7 +108,7 @@ private void generateDocument(StringBuilder sb, String module, JCommander obj) {
             sb.append("\n\n");
             CmdBase cmdObj = (CmdBase) obj.getObjects().get(0);
             cmdObj.jcommander.getCommands().forEach((subK, subV) -> {
-                sb.append("\n\n## ").append(subK).append("\n\n");
+                sb.append("\n\n## ").append(subK).append("\n\n");
                 sb.append(cmdObj.getUsageFormatter().getCommandDescription(subK)).append("\n\n");
                 sb.append("**Command:**\n\n");
                 sb.append("```shell\n$ pulsar-admin ").append(module).append(" ")
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java
index a4deb4444601e..f41e53a80e7f3 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java
@@ -118,7 +118,7 @@ private String generateDocument(String module, JCommander commander) {
             sb.append(" subcommand").append("\n```").append("\n\n");
             cmdObj.getCommands().forEach((subK, subV) -> {
                 if (!subK.equals(name)) {
-                    sb.append("\n\n## ").append(subK).append("\n\n");
+                    sb.append("\n\n## ").append(subK).append("\n\n");
                     String subDesc = cmdObj.getUsageFormatter().getCommandDescription(subK);
                     if (null != subDesc && !subDesc.isEmpty()) {
                         sb.append(subDesc).append("\n");

From 51c2bb4dfbd088810a76963ba86afeec2b193776 Mon Sep 17 00:00:00 2001
From: houxiaoyu 
Date: Mon, 12 Jun 2023 18:42:49 +0800
Subject: [PATCH 402/404] [improve][broker] Choose random thread for
 consumerFlow in PersistentDispatcherSingleActiveConsumer (#20522)

### Motivation

Currently, all subscriptions of one topic will do `consuemrFlow` action in a single thread, which is chosen by topicName:
```
this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topicName);
```

If there is a large number of subscriptions in a topic,  all the work will focus on one thread ---- the chosen thread, which will reduce the consume performance.  So this this patch , I'd like to choose a ramdom thread for `consumerFlow` in `PersistentDispatcherSingleActiveConsumer` to improve the consume performance.

### Modifications

*  `topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topicName);` -> `topic.getBrokerService().getTopicOrderedExecutor().chooseThread();`
*  `this.topicExecutor ` -> `this.executor`
---
 ...PersistentDispatcherSingleActiveConsumer.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
index 7cbf7bd2c787a..d9d0f6adc87ae 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
@@ -61,7 +61,7 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher
 
     private final AtomicBoolean isRescheduleReadInProgress = new AtomicBoolean(false);
     protected final PersistentTopic topic;
-    protected final Executor topicExecutor;
+    protected final Executor executor;
 
     protected final String name;
     private Optional dispatchRateLimiter = Optional.empty();
@@ -79,7 +79,7 @@ public PersistentDispatcherSingleActiveConsumer(ManagedCursor cursor, SubType su
         super(subscriptionType, partitionIndex, topic.getName(), subscription,
                 topic.getBrokerService().pulsar().getConfiguration(), cursor);
         this.topic = topic;
-        this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topicName);
+        this.executor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread();
         this.name = topic.getName() + " / " + (cursor.getName() != null ? Codec.decode(cursor.getName())
                 : ""/* NonDurableCursor doesn't have name */);
         this.readBatchSize = serviceConfig.getDispatcherMaxReadBatchSize();
@@ -148,7 +148,7 @@ protected void cancelPendingRead() {
 
     @Override
     public void readEntriesComplete(final List entries, Object obj) {
-        topicExecutor.execute(() -> internalReadEntriesComplete(entries, obj));
+        executor.execute(() -> internalReadEntriesComplete(entries, obj));
     }
 
     public synchronized void internalReadEntriesComplete(final List entries, Object obj) {
@@ -226,7 +226,7 @@ protected void dispatchEntriesToConsumer(Consumer currentConsumer, List e
                             sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes());
 
                     // Schedule a new read batch operation only after the previous batch has been written to the socket.
-                    topicExecutor.execute(() -> {
+                    executor.execute(() -> {
                             synchronized (PersistentDispatcherSingleActiveConsumer.this) {
                                 Consumer newConsumer = getActiveConsumer();
                                 readMoreEntries(newConsumer);
@@ -238,7 +238,7 @@ protected void dispatchEntriesToConsumer(Consumer currentConsumer, List e
 
     @Override
     public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) {
-        topicExecutor.execute(() -> internalConsumerFlow(consumer));
+        executor.execute(() -> internalConsumerFlow(consumer));
     }
 
     private synchronized void internalConsumerFlow(Consumer consumer) {
@@ -267,7 +267,7 @@ private synchronized void internalConsumerFlow(Consumer consumer) {
 
     @Override
     public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) {
-        topicExecutor.execute(() -> internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch));
+        executor.execute(() -> internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch));
     }
 
     private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) {
@@ -459,7 +459,7 @@ protected Pair calculateToRead(Consumer consumer) {
 
     @Override
     public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
-        topicExecutor.execute(() -> internalReadEntriesFailed(exception, ctx));
+        executor.execute(() -> internalReadEntriesFailed(exception, ctx));
     }
 
     private synchronized void internalReadEntriesFailed(ManagedLedgerException exception, Object ctx) {
@@ -507,7 +507,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep
         topic.getBrokerService().executor().schedule(() -> {
 
             // Jump again into dispatcher dedicated thread
-            topicExecutor.execute(() -> {
+            executor.execute(() -> {
                 synchronized (PersistentDispatcherSingleActiveConsumer.this) {
                     Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
                     // we should retry the read if we have an active consumer and there is no pending read

From dfa2e1b8d503d22664813a03ae2fc1bde4b2bf81 Mon Sep 17 00:00:00 2001
From: tison 
Date: Tue, 13 Jun 2023 08:23:17 +0800
Subject: [PATCH 403/404] fix compile

Signed-off-by: tison 
---
 .../kubernetes/KubernetesRuntimeTest.java     | 45 +++++++++----------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index 811954c470222..fdf9f722aef83 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -35,9 +35,21 @@
 import io.kubernetes.client.openapi.models.V1Service;
 import io.kubernetes.client.openapi.models.V1StatefulSet;
 import io.kubernetes.client.openapi.models.V1Toleration;
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.net.HttpURLConnection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import okhttp3.Call;
+import okhttp3.Response;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.SystemUtils;
 import org.apache.commons.lang3.JavaVersion;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.functions.instance.AuthenticationConfig;
 import org.apache.pulsar.functions.instance.InstanceConfig;
@@ -53,29 +65,15 @@
 import org.apache.pulsar.functions.worker.ConnectorsManager;
 import org.apache.pulsar.functions.worker.FunctionsManager;
 import org.apache.pulsar.functions.worker.WorkerConfig;
+import org.mockito.ArgumentMatcher;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.net.HttpURLConnection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import okhttp3.Call;
-import okhttp3.Response;
-
 import static org.apache.pulsar.functions.runtime.RuntimeUtils.FUNCTIONS_INSTANCE_CLASSPATH;
 import static org.apache.pulsar.functions.utils.FunctionCommon.roundDecimal;
-import org.mockito.ArgumentMatcher;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -84,9 +82,9 @@
 import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotEquals;
@@ -1311,9 +1309,8 @@ private void assertMetricsPortConfigured(Map functionRuntimeFact
     @Test
     public void testDeleteStatefulSetWithTranslatedKubernetesLabelChars() throws Exception {
         InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false);
-        config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false, (fb) -> {
-            return fb.setTenant("c:tenant").setNamespace("c:ns").setName("c:fn");
-        }));
+        config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false,
+                (fb) -> fb.setTenant("c:tenant").setNamespace("c:ns").setName("c:fn")));
 
         CoreV1Api coreApi = mock(CoreV1Api.class);
         AppsV1Api appsApi = mock(AppsV1Api.class);
@@ -1331,9 +1328,7 @@ public void testDeleteStatefulSetWithTranslatedKubernetesLabelChars() throws Exc
         factory.setCoreClient(coreApi);
         factory.setAppsClient(appsApi);
 
-        ArgumentMatcher hasTranslatedFunctionName = (String t) -> {
-            return t.startsWith(expectedFunctionNamePrefix);
-        };
+        ArgumentMatcher hasTranslatedFunctionName = (String t) -> t.startsWith(expectedFunctionNamePrefix);
         
         when(appsApi.deleteNamespacedStatefulSetCall(
                 argThat(hasTranslatedFunctionName),
@@ -1341,8 +1336,8 @@ public void testDeleteStatefulSetWithTranslatedKubernetesLabelChars() throws Exc
 
         ApiException notFoundException = mock(ApiException.class);
         when(notFoundException.getCode()).thenReturn(HttpURLConnection.HTTP_NOT_FOUND);
-        when(appsApi.readNamespacedStatefulSet(argThat(hasTranslatedFunctionName), anyString(),
-                isNull(), isNull(), isNull())).thenThrow(notFoundException);
+        when(appsApi.readNamespacedStatefulSet(
+                argThat(hasTranslatedFunctionName), anyString(), anyString())).thenThrow(notFoundException);
 
         V1PodList podList = mock(V1PodList.class);
         when(podList.getItems()).thenReturn(Collections.emptyList());

From 83c57525c0e9d405d2a6a47e89070132595c6c20 Mon Sep 17 00:00:00 2001
From: tison 
Date: Tue, 13 Jun 2023 08:24:44 +0800
Subject: [PATCH 404/404] fix capture

Signed-off-by: tison 
---
 .../functions/runtime/kubernetes/KubernetesRuntimeTest.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index fdf9f722aef83..7fa279bc1d253 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -1337,7 +1337,7 @@ public void testDeleteStatefulSetWithTranslatedKubernetesLabelChars() throws Exc
         ApiException notFoundException = mock(ApiException.class);
         when(notFoundException.getCode()).thenReturn(HttpURLConnection.HTTP_NOT_FOUND);
         when(appsApi.readNamespacedStatefulSet(
-                argThat(hasTranslatedFunctionName), anyString(), anyString())).thenThrow(notFoundException);
+                argThat(hasTranslatedFunctionName), anyString(), isNull())).thenThrow(notFoundException);
 
         V1PodList podList = mock(V1PodList.class);
         when(podList.getItems()).thenReturn(Collections.emptyList());