diff --git a/docker-compose.yaml b/docker-compose.yaml index 0a5f2efe8..4c42e80f4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -31,37 +31,32 @@ services: networks: - rtr - zookeeper: - image: confluentinc/cp-zookeeper:7.3.0 - hostname: rtr-zookeeper - container_name: rtr-zookeeper - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - networks: - - rtr - kafka: - image: confluentinc/cp-kafka:7.3.0 + image: confluentinc/cp-kafka:7.6.9 hostname: kafka container_name: kafka - depends_on: - - zookeeper ports: - "9092:9092" environment: - KAFKA_BROKER_ID: 1 - KAFKA_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://kafka:9092" - KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://localhost:9092" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" - KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" + # KRaft Configuration + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + CLUSTER_ID: RwB4L8JMReqMdSiPoP-9Og # generated with docker run --rm confluentinc/cp-kafka:7.3.0 kafka-storage random-uuid + + # Listener Configuration + KAFKA_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://kafka:9092,CONTROLLER://kafka:29093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + + # Topic Configuration KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + networks: - rtr @@ -95,7 +90,6 @@ services: container_name: rtr-debezium depends_on: - kafka - - zookeeper ports: - "8085:8083" environment: diff --git a/documentation/DevSetup.md b/documentation/DevSetup.md index 4a0aa363e..4d4c935a1 100644 --- a/documentation/DevSetup.md +++ b/documentation/DevSetup.md @@ -9,7 +9,6 @@ 1. RTR User creation scripts applied 1. Change Data Capture (CDC) enabled for relevant databases and tables 1. [liquibase](../liquibase-service/Dockerfile.local) - Liquibase container with all migration scripts copied from [liquibase-service/src/main/resources/db/](../liquibase-service/src/main/resources/db/). Configured to automatically apply migrations and 1 time onboarding scripts and then close. Entrypoint: [migrate.sh](../containers/liquibase/migrate.sh) -1. [zookeeper](../docker-compose.yaml) - Manages the kafka cluster 1. [kafka](../docker-compose.yaml) - Message broker 1. [kafka-connect](../containers/kafka-connect/Dockerfile) - Reads from the `nrt_*` topics and inserts into `rdb_modern` tables. Requires POST of [mssql-connector.json](../containers/kafka-connect/mssql-connector.json) after container start up. 1. [debezium](../docker-compose.yaml) - Reads Change Data Capture logs and posts messages to Kafka. Requires POST for each connector to be sent after container start up. @@ -36,10 +35,10 @@ docker compose up mssql liquibase -d Wait on liquibase container to complete migration and run onboarding scripts. Container will stop when complete. -### Build zookeeper, kafka, kafka-connect, debezium containers +### Build kafka, kafka-connect, debezium containers ```sh -docker compose up zookeeper kafka kafka-connect debezium -d +docker compose up kafka kafka-connect debezium -d ``` ### Enable connectors for debezium diff --git a/liquibase-service/build.sh b/liquibase-service/build.sh index a8e6ef8ac..acd27f2fa 100755 --- a/liquibase-service/build.sh +++ b/liquibase-service/build.sh @@ -16,11 +16,7 @@ BASE="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" echo "Building SQL Server database" docker-compose -f $BASE/docker-compose.yml up nbs-dataaccess-mssql --build -d -# Build and deploy Zookeeper container -echo "Building Zookeeper" -docker-compose -f $BASE/docker-compose.yml up zookeeper --build -d - -# Build and deploy Kafka Broke container +# Build and deploy Kafka Broker container echo "Building Kafka Broker" docker-compose -f $BASE/docker-compose.yml up broker --build -d @@ -45,4 +41,4 @@ echo "Database: localhost:1433" echo "DB user: sa" echo "DB password: PizzaIsGood33!" echo "Kafka: localhost:9092" -echo "" \ No newline at end of file +echo ""