From 07794e58dd51797ac0ed585709279212280e08c3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 5 Nov 2019 04:55:32 +0100 Subject: [PATCH 1/4] Comeback for docs.docker.com/docker-hub/builds/automated-testing/ --- ...ker-compose.yml => docker-compose.test.yml | 86 +++---------------- {build-contracts/pixy => pixy}/Dockerfile | 0 {build-contracts/pixy => pixy}/pixy.yaml | 0 3 files changed, 12 insertions(+), 74 deletions(-) rename build-contracts/docker-compose.yml => docker-compose.test.yml (57%) rename {build-contracts/pixy => pixy}/Dockerfile (100%) rename {build-contracts/pixy => pixy}/pixy.yaml (100%) diff --git a/build-contracts/docker-compose.yml b/docker-compose.test.yml similarity index 57% rename from build-contracts/docker-compose.yml rename to docker-compose.test.yml index cc2ceeb8..e1d294da 100644 --- a/build-contracts/docker-compose.yml +++ b/docker-compose.test.yml @@ -2,13 +2,13 @@ version: "3.4" services: zookeeper: - image: solsson/kafka:2.2.0@sha256:cf048d6211b6b48f1783f97cb41add511386e2f0a5f5c62fa0eee9564dcd3e9a + image: solsson/kafka@sha256:e03747d2f1971d7efd91f70d2522c4fa874853f931d02f05015ef0976fba849f entrypoint: ./bin/zookeeper-server-start.sh command: - ./config/zookeeper.properties kafka: - image: solsson/kafka:2.2.0@sha256:cf048d6211b6b48f1783f97cb41add511386e2f0a5f5c62fa0eee9564dcd3e9a + image: solsson/kafka@sha256:e03747d2f1971d7efd91f70d2522c4fa874853f931d02f05015ef0976fba849f depends_on: - zookeeper entrypoint: @@ -17,6 +17,8 @@ services: - --override - zookeeper.connect=zookeeper:2181 - --override + - auto.create.topics.enable=false + - --override - log.retention.hours=-1 - --override - log.dirs=/var/lib/kafka/data/topics @@ -47,7 +49,7 @@ services: - 0.0.0.0:19090 topic1-create: - image: solsson/kafka:2.2.0@sha256:cf048d6211b6b48f1783f97cb41add511386e2f0a5f5c62fa0eee9564dcd3e9a + image: solsson/kafka-cli@sha256:9fa3306e9f5d18283d10e01f7c115d8321eedc682f262aff784bd0126e1f2221 depends_on: - kafka entrypoint: @@ -64,34 +66,22 @@ services: - --replication-factor - '1' - kkv-plainjava: - image: yolean/kafka-keyvalue:plainjava - build: - context: ../ - target: runtime-plainjava - labels: - - com.yolean.build-target - command: - - echo - - This was only a build. Exiting. - - cache1: + sut: depends_on: - kafka - topic1-create - image: yolean/kafka-keyvalue:latest + #build: . build: - context: ../ - labels: - - com.yolean.build-target + context: . + target: runtime-plainjava ports: - 19081:8080 environment: - kafka_bootstrap=kafka:9092 - kafka_group_id=cache1 - kafka_offset_reset=latest - - topics=topic1 - #- update_targets=http://example-nodejs-client:8081/kafka-keyvalue/v1/updates,http://onupdate-logging:8080/,http://example-nodejs-client:8082/testpost + - topic=topic1 + #- target=http://example-nodejs-client:8081/kafka-keyvalue/v1/updates,http://onupdate-logging:8080/,http://example-nodejs-client:8082/testpost onupdate-logging: image: solsson/httpstatus-random:access-logging@sha256:5330b2e84457e65ae9552acb00b26e8b7b2a878bcafb3baabddc13c1f840cd90 @@ -102,7 +92,7 @@ services: depends_on: - onupdate-logging - pixy - - cache1 + - sut image: solsson/curl@sha256:92ebf15ac57bea360484480336ed5d9fa16d38d773fd00f7e9fb2cae94baf25a labels: - com.yolean.build-contract @@ -122,55 +112,3 @@ services: curl --ipv4 -d '{"x":2}' -H 'Content-Type: application/json' 'http://pixy:19090/topics/topic1/messages?key=smoketest1&sync' -f curl --ipv4 -d '{"x":3}' -H 'Content-Type: application/json' 'http://pixy:19090/topics/topic1/messages?key=smoketest1&sync' -f curl --ipv4 --retry 5 --retry-connrefused http://cache1:19081/cache/v1/raw/smoketest1 -f | grep '{"x":3}' - - example-nodejs-client: - depends_on: - - pixy - - cache1 - build: - context: ../example-nodejs-client - labels: - - com.yolean.build-contract - expose: - - "8081" - - "8082" - environment: - - PIXY_HOST=http://pixy:19090 - - CACHE1_HOST=http://cache1:19081 - command: - # By design the cache service deals with a single topic, meaning that tests probably can't run concurrently - - --runInBand - - # compose build && compose up -d k8s-kind && compose logs -f k8s-kind - # compose exec k8s-kind cat /root/.kube/kind-config-kkv > ./kubeconfig-kkv-kind - # export KUBECONFIG=$(pwd)/kubeconfig-kkv-kind - # kubectl apply -f kontrakt/ - k8s-kind: - build: - context: ../kontrakt/kind - volumes: - - /var/run/docker.sock:/var/run/docker.sock:rw - network_mode: host - environment: - - KIND_NAME=kkv - - KUBECONFIG=/root/.kube/kind-config-kkv - command: - - /bin/sh - - -cex - - | - kind delete cluster --name=$$KIND_NAME || echo "Found no prior cluster" - kind create cluster --name=$$KIND_NAME - kind get kubeconfig-path --name=$$KIND_NAME - kubectl cluster-info - - cd kubernetes-kafka - kubectl apply -f 00-namespace.yml - kubectl apply -f rbac-namespace-default/ - kubectl apply -k variants/scale-1-ephemeral/ - kubectl apply -f pixy/ - - # Load images built by docker-compose (to re-load use compose exec) - kind load docker-image yolean/kafka-keyvalue:latest --name=$$KIND_NAME - kind load docker-image yolean/kafka-keyvalue:plainjava --name=$$KIND_NAME - - tail -f /dev/null diff --git a/build-contracts/pixy/Dockerfile b/pixy/Dockerfile similarity index 100% rename from build-contracts/pixy/Dockerfile rename to pixy/Dockerfile diff --git a/build-contracts/pixy/pixy.yaml b/pixy/pixy.yaml similarity index 100% rename from build-contracts/pixy/pixy.yaml rename to pixy/pixy.yaml From 710b865e2d1846e9df71e4eba78c7d6230c6d3a5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 5 Nov 2019 04:58:16 +0100 Subject: [PATCH 2/4] Now we could actually complete a test, but we need to exit properly --- docker-compose.test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index e1d294da..0189dfa5 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -82,6 +82,7 @@ services: - kafka_offset_reset=latest - topic=topic1 #- target=http://example-nodejs-client:8081/kafka-keyvalue/v1/updates,http://onupdate-logging:8080/,http://example-nodejs-client:8082/testpost + - max_polls=10 onupdate-logging: image: solsson/httpstatus-random:access-logging@sha256:5330b2e84457e65ae9552acb00b26e8b7b2a878bcafb3baabddc13c1f840cd90 From 3b4abe8bd1dfd8c9a2cdeea99957b19d1a1d5d08 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 5 Nov 2019 05:08:54 +0100 Subject: [PATCH 3/4] We need to revisit shutdown behavior, CLI style? --- .../java/se/yolean/kafka/keyvalue/ConsumerAtLeastOnce.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/se/yolean/kafka/keyvalue/ConsumerAtLeastOnce.java b/src/main/java/se/yolean/kafka/keyvalue/ConsumerAtLeastOnce.java index 7543c5ef..b8328418 100644 --- a/src/main/java/se/yolean/kafka/keyvalue/ConsumerAtLeastOnce.java +++ b/src/main/java/se/yolean/kafka/keyvalue/ConsumerAtLeastOnce.java @@ -216,6 +216,9 @@ public void run() { } finally { logger.info("Closing consumer ..."); consumer.close(); + // Before we introduce proper termination support let's await: https://github.com/quarkusio/quarkus/issues/284 + // https://github.com/quarkusio/quarkus/blob/0.28.1/core/runtime/src/main/java/io/quarkus/runtime/Application.java#L121 + // https://github.com/quarkusio/quarkus/issues/2150 https://github.com/quarkusio/quarkus/issues/2851 https://github.com/quarkusio/quarkus/issues/3296 https://github.com/quarkusio/quarkus/pull/4924/files logger.info("Consumer closed at stage {}; Use liveness probes with /health for app termination", stage); } } From 1bfe25dd9c888f054bd19cf5a73f92752814c343 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 5 Nov 2019 05:58:38 +0100 Subject: [PATCH 4/4] Documents a docker-compose example --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d961b9b9..43f3dcba 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ kubectl apply -f https://github.com/Yolean/kubernetes-kafka/raw/50345f266287861d SKAFFOLD_NO_PRUNE=true skaffold dev ``` +### docker-compose + +We maintain a docker-compose file for [Docker Hub automated test](https://docs.docker.com/docker-hub/builds/automated-testing/). +It can be used for dev too, for example: +`docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml up --build topic1-create sut` + ## Logging See [Quarkus' logging configuration](https://quarkus.io/guides/logging-guide).