From c9cfd132d3690232ffd1b89d47d7bce7bc044d50 Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Mon, 28 Oct 2024 23:30:44 -0300 Subject: [PATCH 1/9] add sorting-hat as git submodule --- .gitmodules | 3 +++ sorting-hat | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 sorting-hat diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..26a09f6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sorting-hat"] + path = sorting-hat + url = git@github.com:the-sortinghat/backend.git diff --git a/sorting-hat b/sorting-hat new file mode 160000 index 0000000..95695a1 --- /dev/null +++ b/sorting-hat @@ -0,0 +1 @@ +Subproject commit 95695a1808e477ce67d8257891f4fa9be21f8cce From 726a18a682cb24197822c520cda69676b9c73682 Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Tue, 29 Oct 2024 22:00:11 -0300 Subject: [PATCH 2/9] include sorting-hat execution in docker-compose.yaml --- docker-compose.yaml | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3031e56..c6b2883 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,24 +1,41 @@ version: '3.8' services: - web: + us_vision: build: . env_file: - ./.env.docker-compose ports: - - "8080:8080" + - "8081:8080" depends_on: - - mongodb + - us_vision_mongodb - mongodb: + us_vision_mongodb: image: mongo env_file: - ./.env.docker-compose volumes: - - mongodb_data:/data/db + - us_vision_mongodb_data:/data/db - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro + ports: + - "27018:27017" + + sorting_hat_mongodb: + image: mongo:4.2-bionic ports: - "27017:27017" + volumes: + - sorting_hat_mongodb_data:/data/db + + sorting_hat: + build: ./sorting-hat + ports: + - "8080:8080" + env_file: + - ./sorting-hat/.env + depends_on: + - sorting_hat_mongodb volumes: - mongodb_data: + us_vision_mongodb_data: + sorting_hat_mongodb_data: From 6a1074946e9616901cd09b6acb5509f5b8f021ba Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Tue, 29 Oct 2024 23:11:12 -0300 Subject: [PATCH 3/9] add kafka images to docker-compose.yaml --- docker-compose.yaml | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index c6b2883..0004d22 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -36,6 +36,55 @@ services: depends_on: - sorting_hat_mongodb + zookeeper: + image: confluentinc/cp-zookeeper:7.2.2 + hostname: zookeeper + container_name: zookeeper + networks: + - localnet + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker: + image: confluentinc/cp-kafka:7.2.2 + hostname: broker + container_name: broker + depends_on: + - zookeeper + networks: + - localnet + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_LISTENERS: PLAINTEXT://broker:29092,LISTENER_2://broker:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,LISTENER_2://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_2:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + CONFLUENT_SUPPORT_CUSTOMER_ID: "anonymous" + KAFKA_DELETE_TOPIC_ENABLE: "true" + + schema-registry: + image: confluentinc/cp-schema-registry:7.2.2 + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + networks: + - localnet + ports: + - "3081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "broker:29092" + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 volumes: us_vision_mongodb_data: sorting_hat_mongodb_data: + +networks: + localnet: + attachable: true From e8996fcb87a225c1ee8d9511e59ed76c7575b6c0 Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Wed, 30 Oct 2024 23:55:33 -0300 Subject: [PATCH 4/9] add connect image to docker-compose.yaml --- docker-compose.yaml | 37 ++++++++++++++++++++++++++++++++ mongodb-kafka-connect.Dockerfile | 5 +++++ 2 files changed, 42 insertions(+) create mode 100644 mongodb-kafka-connect.Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml index 0004d22..5f61a2d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -81,6 +81,43 @@ services: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "broker:29092" SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + build: + context: . + dockerfile: mongodb-kafka-connect.Dockerfile + ports: + - "35000:35000" + - "8083:8083" + hostname: connect + container_name: connect + depends_on: + - zookeeper + - broker + networks: + - localnet + environment: + KAFKA_JMX_PORT: 35000 + KAFKA_JMX_HOSTNAME: localhost + CONNECT_BOOTSTRAP_SERVERS: "broker:29092" + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: connect-cluster-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_ZOOKEEPER_CONNECT: "zookeeper:2181" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_CONNECTIONS_MAX_IDLE_MS: 180000 + CONNECT_METADATA_MAX_AGE_MS: 180000 + CONNECT_AUTO_CREATE_TOPICS_ENABLE: "true" + CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + volumes: us_vision_mongodb_data: sorting_hat_mongodb_data: diff --git a/mongodb-kafka-connect.Dockerfile b/mongodb-kafka-connect.Dockerfile new file mode 100644 index 0000000..25af83f --- /dev/null +++ b/mongodb-kafka-connect.Dockerfile @@ -0,0 +1,5 @@ +FROM confluentinc/cp-kafka-connect:7.2.5 + +RUN confluent-hub install --no-prompt --verbose mongodb/kafka-connect-mongodb:latest + +ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components" From 5a5952975d72ef74842d0e7a1bbe6f5efc1bc29c Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Fri, 1 Nov 2024 11:26:09 -0300 Subject: [PATCH 5/9] remove network from docker-compose.yaml --- docker-compose.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5f61a2d..38e785c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -40,8 +40,6 @@ services: image: confluentinc/cp-zookeeper:7.2.2 hostname: zookeeper container_name: zookeeper - networks: - - localnet environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 @@ -52,8 +50,6 @@ services: container_name: broker depends_on: - zookeeper - networks: - - localnet environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" @@ -72,8 +68,6 @@ services: container_name: schema-registry depends_on: - broker - networks: - - localnet ports: - "3081:8081" environment: @@ -94,8 +88,6 @@ services: depends_on: - zookeeper - broker - networks: - - localnet environment: KAFKA_JMX_PORT: 35000 KAFKA_JMX_HOSTNAME: localhost @@ -121,7 +113,3 @@ services: volumes: us_vision_mongodb_data: sorting_hat_mongodb_data: - -networks: - localnet: - attachable: true From 8601d0e91b01209b375425fbbbc12be8d19af625 Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Fri, 1 Nov 2024 19:26:08 -0300 Subject: [PATCH 6/9] add replica set settings --- config-replica.js | 6 ++++++ docker-compose.yaml | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 config-replica.js diff --git a/config-replica.js b/config-replica.js new file mode 100644 index 0000000..00e7f2f --- /dev/null +++ b/config-replica.js @@ -0,0 +1,6 @@ +rsconf = { + _id: "rs0", + members: [{ _id: 0, host: "sorting_hat_mongodb:27017", priority: 1.0 }], +}; +rs.initiate(rsconf); +rs.status(); diff --git a/docker-compose.yaml b/docker-compose.yaml index 38e785c..3bc828e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -26,6 +26,22 @@ services: - "27017:27017" volumes: - sorting_hat_mongodb_data:/data/db + command: --replSet rs0 --oplogSize 128 + + + sorting_hat_mongodb-setup: + image: mongo + depends_on: + - sorting_hat_mongodb + volumes: + - ./config-replica.js:/config-replica.js + entrypoint: + [ + "bash", + "-c", + "sleep 10 && mongosh --host sorting_hat_mongodb:27017 config-replica.js && sleep 10", + ] + restart: "no" sorting_hat: build: ./sorting-hat @@ -35,6 +51,7 @@ services: - ./sorting-hat/.env depends_on: - sorting_hat_mongodb + - sorting_hat_mongodb-setup zookeeper: image: confluentinc/cp-zookeeper:7.2.2 From 545cf45bff63ee3f2691656ef2aac7a97ada0c58 Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Fri, 1 Nov 2024 20:53:43 -0300 Subject: [PATCH 7/9] add kafka rest-proxy container --- docker-compose.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3bc828e..5a70da7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -93,6 +93,22 @@ services: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + rest-proxy: + image: confluentinc/cp-kafka-rest:7.2.2 + depends_on: + - zookeeper + - broker + - schema-registry + ports: + - "3082:8082" + hostname: rest-proxy + container_name: rest-proxy + environment: + KAFKA_REST_HOST_NAME: rest-proxy + KAFKA_REST_BOOTSTRAP_SERVERS: "broker:29092" + KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" + KAFKA_REST_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + connect: build: context: . From 0959b3a99448186df512d8fc83ee65e440415c38 Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Mon, 4 Nov 2024 23:56:18 -0300 Subject: [PATCH 8/9] create app async module --- app-async/build.gradle.kts | 35 +++++++++++++++++++ app-async/gradle.properties | 4 +++ .../com/usvision/kafka/CompanySystemConfig.kt | 35 +++++++++++++++++++ settings.gradle.kts | 1 + 4 files changed, 75 insertions(+) create mode 100644 app-async/build.gradle.kts create mode 100644 app-async/gradle.properties create mode 100644 app-async/src/main/kotlin/com/usvision/kafka/CompanySystemConfig.kt diff --git a/app-async/build.gradle.kts b/app-async/build.gradle.kts new file mode 100644 index 0000000..e892588 --- /dev/null +++ b/app-async/build.gradle.kts @@ -0,0 +1,35 @@ +val mockk_version: String by project +val kafka_clients_version: String by project +val kafka_json_serializer_version: String by project + + +plugins { + application + kotlin("jvm") + id("com.github.johnrengelman.shadow") version "7.0.0" +} + +group = "com.usvision.async" +version = "0.0.1" + +repositories { + mavenCentral() + maven(url = "https://packages.confluent.io/maven/") +} + +tasks.test { + useJUnitPlatform() +} + +dependencies { + implementation(project(":app-creation")) + implementation(project(":app-model")) + implementation(project(":app-persistence")) + + implementation("org.apache.kafka:kafka-clients:${kafka_clients_version}") + implementation("io.confluent:kafka-json-serializer:${kafka_json_serializer_version}") + + + testImplementation("io.mockk:mockk:${mockk_version}") + testImplementation(kotlin("test")) +} \ No newline at end of file diff --git a/app-async/gradle.properties b/app-async/gradle.properties new file mode 100644 index 0000000..bca845c --- /dev/null +++ b/app-async/gradle.properties @@ -0,0 +1,4 @@ +kotlin.code.style=official +mockk_version=1.12.3 +kafka_clients_version=3.3.1 +kafka_json_serializer_version=5.0.1 \ No newline at end of file diff --git a/app-async/src/main/kotlin/com/usvision/kafka/CompanySystemConfig.kt b/app-async/src/main/kotlin/com/usvision/kafka/CompanySystemConfig.kt new file mode 100644 index 0000000..776400e --- /dev/null +++ b/app-async/src/main/kotlin/com/usvision/kafka/CompanySystemConfig.kt @@ -0,0 +1,35 @@ +package com.usvision.kafka + +import org.apache.kafka.clients.consumer.KafkaConsumer +import java.time.Duration + + +fun main() { + val consumerConfigs = + mapOf( + "bootstrap.servers" to "localhost:9092", + "auto.offset.reset" to "earliest", + "key.deserializer" to "org.apache.kafka.common.serialization.StringDeserializer", + "value.deserializer" to "org.apache.kafka.common.serialization.ByteArrayDeserializer", + "group.id" to "usvision.serviceschema", + "security.protocol" to "PLAINTEXT" + ) + + val topic = "sorting-hat-database.serviceSchema" + val consumer = KafkaConsumer(consumerConfigs) + + consumer.subscribe(listOf(topic)) + + while (true) { + try { + consumer.poll(Duration.ofMillis(400)).forEach { + item -> println(item.value()) + } + } catch (ex: Exception) { + println(ex) + } + } + + + +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index d64241d..6a251d3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -6,3 +6,4 @@ include("app-reports") include("app-persistence") include("app-web") include("app-creation") +include("app-async") From e603201450014b5f1f0ee4a950492291d01476ad Mon Sep 17 00:00:00 2001 From: Vinicius Frota Date: Mon, 4 Nov 2024 23:56:43 -0300 Subject: [PATCH 9/9] expose broker to localhost --- docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5a70da7..a1427be 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -67,10 +67,12 @@ services: container_name: broker depends_on: - zookeeper + ports: + - "9092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENERS: PLAINTEXT://broker:29092,LISTENER_2://broker:9092 + KAFKA_LISTENERS: PLAINTEXT://broker:29092,LISTENER_2://localhost:9092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,LISTENER_2://localhost:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_2:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT