From f372ea6012c55dc1a05176a043328cc9ba14dd81 Mon Sep 17 00:00:00 2001 From: ruihongzhou Date: Mon, 17 Jun 2024 16:47:32 +0800 Subject: [PATCH 1/2] [improve][pip] PIP-362: Add admin APIs for batch updating and deleting bookie rack information. --- pip/pip-362.md | 135 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 pip/pip-362.md diff --git a/pip/pip-362.md b/pip/pip-362.md new file mode 100644 index 0000000000000..f034ac66188c4 --- /dev/null +++ b/pip/pip-362.md @@ -0,0 +1,135 @@ +# PIP-362: Add admin APIs for batch updating and deleting bookie rack information + +# Background knowledge + +Each time the rack information of a bookie is updated or deleted, the mapping information of the bookie rack +will be updated in the `/bookies` directory on Zookeeper. At the same time, all bookies observing this path will be +notified to fetch the latest bookie rack mapping information from Zookeeper. + +# Motivation + +Currently, we only provide single update or delete admin APIs for bookie rack information (`updateBookieRackInfo` +and `deleteBookieRackInfo`). If we want to update the rack information for n bookies, we need to call +`updateBookieRackInfo` n times, which will result in a large number of write and read operations on Zookeeper, +thereby affecting Zookeeper's stability. + +Therefore, we want to introduce admin APIs for batch updating and deleting bookie rack information to reduce +the number of times Zookeeper is accessed when batch setting bookie rack information. + +# Goals + +## In Scope + +Add admin APIs for batch updating and deleting bookie rack information. + +# High Level Design + +Add admin APIs for batch updating and deleting bookie rack information. + +# Detailed Design + +## Design & Implementation Details + +Add admin APIs for batch updating and deleting bookie rack information. + +## Public-facing Changes + +### Public API + +```java +@DELETE +@Path("/racks-info") +@ApiOperation( + value = "Removed the rack placement information for a batch of bookies in the cluster", + notes = "If the 'deleteAll' parameter is set to true, it will remove the rack placement " + + "information for all bookies in the cluster, ignoring the 'bookieAddresses' parameter" +) +@ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") +}) +public void batchDeleteBookiesRackInfo(@Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "List of bookie addresses") + @QueryParam("bookieAddresses") List bookieAddresses, + @ApiParam(value = "Whether to delete all bookies rack info", + defaultValue = "false") + @QueryParam("deleteAll") @DefaultValue("false") + boolean deleteAll) throws Exception { + ... +} + +@POST +@Path("/racks-info") +@ApiOperation(value = "Updates the rack placement information for a batch of bookies in the cluster") +@ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")} +) +public void batchUpdateBookiesRackInfo(@Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "List of bookie info", required = true) + List extBookieInfos) throws Exception { + ... +} +``` + +```java +/** + * Remove rack placement information for a batch of bookies in the cluster. + */ +void batchDeleteBookiesRackInfo(List bookieAddresses) throws PulsarAdminException; + +/** + * Remove rack placement information for a batch of bookies in the cluster asynchronously. + */ +CompletableFuture batchDeleteBookiesRackInfoAsync(List bookieAddresses); + +/** + * Clears the rack placement information for all bookies int the cluster. + */ +void clearAllBookiesRackInfo() throws PulsarAdminException; + +/** + * Clears the rack placement information for all bookies int the cluster asynchronously. + */ +CompletableFuture clearAllBookiesRackInfoAsync(); + +/** + * Updates the rack placement information for a batch of bookies in the cluster. + */ +void batchUpdateBookiesRackInfo(List extBookieInfos) throws PulsarAdminException; + +/** + * Updates the rack placement information for a batch of bookies in the cluster asynchronously. + */ +CompletableFuture batchUpdateBookiesRackInfoAsync(List extBookieInfos); +``` + +### Binary protocol + +### Configuration + +### CLI + +### Metrics + +# Monitoring + +# Security Considerations + +# Backward & Forward Compatibility + +## Revert + +## Upgrade + +# Alternatives + +# General Notes + +# Links + + +* Mailing List discussion thread: +* Mailing List voting thread: From c90c404595a58ea9f4803e8c5f4ab5f91bdf860d Mon Sep 17 00:00:00 2001 From: ruihongzhou Date: Mon, 17 Jun 2024 17:09:47 +0800 Subject: [PATCH 2/2] optimize --- pip/pip-362.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/pip/pip-362.md b/pip/pip-362.md index f034ac66188c4..32371edcc6209 100644 --- a/pip/pip-362.md +++ b/pip/pip-362.md @@ -3,7 +3,7 @@ # Background knowledge Each time the rack information of a bookie is updated or deleted, the mapping information of the bookie rack -will be updated in the `/bookies` directory on Zookeeper. At the same time, all bookies observing this path will be +will be updated in `/bookies` on Zookeeper. At the same time, all bookies observing this path will be notified to fetch the latest bookie rack mapping information from Zookeeper. # Motivation @@ -35,7 +35,89 @@ Add admin APIs for batch updating and deleting bookie rack information. ## Public-facing Changes ### Public API +org.apache.pulsar.common.policies.data.ExtBookieInfo +```java +public interface ExtBookieInfo { + String getAddress(); + String getGroup(); + String getRack(); + String getHostname(); + + interface Builder { + Builder address(String address); + Builder group(String group); + Builder rack(String rack); + Builder hostname(String hostname); + ExtBookieInfo build(); + } + + static Builder builder() { + return ExtBookieInfoImpl.builder(); + } +} +``` + +org.apache.pulsar.common.policies.data.impl.ExtBookieInfoImpl +```java +/** + * Ext Bookie information. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public final class ExtBookieInfoImpl implements ExtBookieInfo { + private String address; + private String group; + private String rack; + private String hostname; + + public static ExtBookieInfoImplBuilder builder() { + return new ExtBookieInfoImplBuilder(); + } + + public static class ExtBookieInfoImplBuilder implements ExtBookieInfo.Builder { + private String address; + private String group; + private String rack; + private String hostname; + private static final String PATH_SEPARATOR = "/"; + + public ExtBookieInfoImplBuilder address(String address) { + this.address = address; + return this; + } + + public ExtBookieInfoImplBuilder group(String group) { + this.group = group; + return this; + } + + public ExtBookieInfoImplBuilder rack(String rack) { + this.rack = rack; + return this; + } + + public ExtBookieInfoImplBuilder hostname(String hostname) { + this.hostname = hostname; + return this; + } + + public ExtBookieInfoImpl build() { + checkArgument(rack != null && !rack.isEmpty() && !rack.equals(PATH_SEPARATOR), + "rack name is invalid, it should not be null, empty or '/'"); + return new ExtBookieInfoImpl(address, group, rack, hostname); + } + + public static void checkArgument(boolean expression, @NonNull Object errorMessage) { + if (!expression) { + throw new IllegalArgumentException(String.valueOf(errorMessage)); + } + } + } +} +``` +V2/Namespaces.java ```java @DELETE @Path("/racks-info") @@ -72,6 +154,7 @@ public void batchUpdateBookiesRackInfo(@Suspended final AsyncResponse asyncRespo } ``` +Client ```java /** * Remove rack placement information for a batch of bookies in the cluster.