local dev without zookeeper

docker-compose.yaml

@@ -31,37 +31,32 @@ services:
     networks:
       - rtr
 
-  zookeeper:
-    image: confluentinc/cp-zookeeper:7.3.0
-    hostname: rtr-zookeeper
-    container_name: rtr-zookeeper
-    ports:
-      - "2181:2181"
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-    networks:
-      - rtr
-
   kafka:
-    image: confluentinc/cp-kafka:7.3.0
+    image: confluentinc/cp-kafka:7.6.9
     hostname: kafka
     container_name: kafka
-    depends_on:
-      - zookeeper
     ports:
       - "9092:9092"
     environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://kafka:9092"
-      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://localhost:9092"
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
-      KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
+      # KRaft Configuration
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      CLUSTER_ID: RwB4L8JMReqMdSiPoP-9Og # generated with docker run --rm confluentinc/cp-kafka:7.3.0 kafka-storage random-uuid
+
+      # Listener Configuration
+      KAFKA_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://kafka:9092,CONTROLLER://kafka:29093
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+
+      # Topic Configuration
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+
     networks:
       - rtr
 
@@ -95,7 +90,6 @@ services:
     container_name: rtr-debezium
     depends_on:
       - kafka
-      - zookeeper
     ports:
       - "8085:8083"
     environment:

documentation/DevSetup.md

@@ -9,7 +9,6 @@
    1. RTR User creation scripts applied
    1. Change Data Capture (CDC) enabled for relevant databases and tables
 1. [liquibase](../liquibase-service/Dockerfile.local) - Liquibase container with all migration scripts copied from [liquibase-service/src/main/resources/db/](../liquibase-service/src/main/resources/db/). Configured to automatically apply migrations and 1 time onboarding scripts and then close. Entrypoint: [migrate.sh](../containers/liquibase/migrate.sh)
-1. [zookeeper](../docker-compose.yaml) - Manages the kafka cluster
 1. [kafka](../docker-compose.yaml) - Message broker
 1. [kafka-connect](../containers/kafka-connect/Dockerfile) - Reads from the `nrt_*` topics and inserts into `rdb_modern` tables. Requires POST of [mssql-connector.json](../containers/kafka-connect/mssql-connector.json) after container start up.
 1. [debezium](../docker-compose.yaml) - Reads Change Data Capture logs and posts messages to Kafka. Requires POST for each connector to be sent after container start up.
@@ -36,10 +35,10 @@ docker compose up mssql liquibase -d
 
 Wait on liquibase container to complete migration and run onboarding scripts. Container will stop when complete.
 
-### Build zookeeper, kafka, kafka-connect, debezium containers
+### Build kafka, kafka-connect, debezium containers
 
 ```sh
-docker compose up zookeeper kafka kafka-connect debezium -d
+docker compose up kafka kafka-connect debezium -d
 ```
 
 ### Enable connectors for debezium

liquibase-service/build.sh

@@ -16,11 +16,7 @@ BASE="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 echo "Building SQL Server database"
 docker-compose -f $BASE/docker-compose.yml up nbs-dataaccess-mssql --build -d
 
-# Build and deploy Zookeeper container
-echo "Building Zookeeper"
-docker-compose -f $BASE/docker-compose.yml up zookeeper --build -d
-
-# Build and deploy Kafka Broke container
+# Build and deploy Kafka Broker container
 echo "Building Kafka Broker"
 docker-compose -f $BASE/docker-compose.yml up broker --build -d
 
@@ -45,4 +41,4 @@ echo "Database: localhost:1433"
 echo "DB user: sa"
 echo "DB password: PizzaIsGood33!"
 echo "Kafka: localhost:9092"
-echo ""
+echo ""