docker-compose.yml

services:
  # Zookeeper pour Kafka
  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - streaming-network
    volumes:
      - ./data/zookeeper:/var/lib/zookeeper/data
      - ./data/zookeeper-logs:/var/lib/zookeeper/log
    healthcheck:
      test: ["CMD", "/bin/sh", "-c", "echo ruok | nc localhost 2181"]
      interval: 10s
      timeout: 5s
      retries: 5

  # Kafka Broker
  kafka:
    image: confluentinc/cp-kafka:7.5.0
    hostname: kafka
    container_name: kafka
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
      KAFKA_CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      KAFKA_CONFLUENT_METRICS_ENABLE: 'false'
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      CLUSTER_ID: 'streaming-cluster'
    networks:
      - streaming-network
    volumes:
      - ./data/kafka:/var/lib/kafka/data
      - ./kafka-init.sh:/usr/local/bin/kafka-init.sh
    entrypoint:
      - /bin/bash
      - -c
      - |
        rm -rf /var/lib/kafka/data
        mkdir -p /var/lib/kafka/data
        exec /etc/confluent/docker/run
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:9092"]
      interval: 10s
      timeout: 5s
      retries: 5

  # Schema Registry
  schema-registry:
    image: confluentinc/cp-schema-registry:7.5.0
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    networks:
      - streaming-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
      interval: 10s
      timeout: 5s
      retries: 5

  # Kafka Connect
  kafka-connect:
    image: confluentinc/cp-kafka-connect:7.5.0
    hostname: kafka-connect
    container_name: kafka-connect
    depends_on:
      schema-registry:
        condition: service_healthy
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka:29092'
      CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
    volumes:
      - ./data/input:/data/input
      - ./data/processed:/data/processed
    networks:
      - streaming-network

  # Apache Flink JobManager
  flink-jobmanager:
    image: flink:1.18-scala_2.12
    hostname: flink-jobmanager
    container_name: flink-jobmanager
    ports:
      - "8088:8081"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        state.backend: filesystem
        state.checkpoints.dir: file:///tmp/flink-checkpoints
        state.savepoints.dir: file:///tmp/flink-savepoints
    networks:
      - streaming-network
    volumes:
      - ./flink-jobs:/opt/flink/jobs

  # Apache Flink TaskManager
  flink-taskmanager:
    image: flink:1.18-scala_2.12
    hostname: flink-taskmanager
    depends_on:
      - flink-jobmanager
    command: taskmanager
    deploy:
      replicas: 2
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.numberOfTaskSlots: 4
        state.backend: filesystem
        state.checkpoints.dir: file:///tmp/flink-checkpoints
    networks:
      - streaming-network

  # ClickHouse
  clickhouse:
    image: clickhouse/clickhouse-server:23.8
    hostname: clickhouse
    container_name: clickhouse
    ports:
      - "8123:8123"
      - "9000:9000"
    environment:
      CLICKHOUSE_DB: streaming_demo
      CLICKHOUSE_USER: admin
      CLICKHOUSE_PASSWORD: admin
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - ./clickhouse/init:/docker-entrypoint-initdb.d
      - ./clickhouse/config.xml:/etc/clickhouse-server/config.d/cors-headers.xml
      - ./data/clickhouse:/var/lib/clickhouse
    networks:
      - streaming-network
    ulimits:
      nofile:
        soft: 262144
        hard: 262144

  # Apache Superset (BI & OLAP)
  superset:
    image: apache/superset:3.0.0
    hostname: superset
    container_name: superset
    ports:
      - "8089:8088"
    environment:
      SUPERSET_SECRET_KEY: 'streaming_first_demo_secret_key_change_in_prod'
      SUPERSET_LOAD_EXAMPLES: 'no'
    volumes:
      - ./superset/superset_config.py:/app/pythonpath/superset_config.py
      - ./superset/init-superset.sh:/app/docker/init-superset.sh
      - ./data/superset:/app/superset_home
    networks:
      - streaming-network
    depends_on:
      - clickhouse
    command: >
      bash -c "
      pip install clickhouse-connect &&
      /app/docker/docker-bootstrap.sh app-gunicorn
      "

  # File Generator (simulation de données - génère CSV/JSON)
  file-generator:
    build:
      context: ./file-generator
      dockerfile: Dockerfile
    container_name: file-generator
    depends_on:
      schema-registry:
        condition: service_healthy
    volumes:
      - ./data/input:/output
    environment:
      OUTPUT_DIR: /output
      GENERATION_INTERVAL: 30
    networks:
      - streaming-network

  # CSV to Kafka Avro Producer (lit CSV et envoie en Avro à Kafka)
  csv-to-kafka-producer:
    build:
      context: ./csv-to-kafka-producer
      dockerfile: Dockerfile
    container_name: csv-to-kafka-producer
    depends_on:
      schema-registry:
        condition: service_healthy
      file-generator:
        condition: service_started
    volumes:
      - ./data/input:/data/input
      - ./data/processed:/data/processed
      - ./schemas:/schemas:ro
    environment:
      KAFKA_BOOTSTRAP_SERVERS: kafka:29092
      SCHEMA_REGISTRY_URL: http://schema-registry:8081
      INPUT_DIR: /data/input
      PROCESSED_DIR: /data/processed
      POLL_INTERVAL: 5
    networks:
      - streaming-network

  # Streaming Real-time Avro Producer (génère et envoie en Avro temps réel)
  streaming-producer:
    build:
      context: ./streaming-producer
      dockerfile: Dockerfile
    container_name: streaming-producer
    depends_on:
      schema-registry:
        condition: service_healthy
    volumes:
      - ./schemas:/schemas:ro
    environment:
      KAFKA_BOOTSTRAP_SERVERS: kafka:29092
      SCHEMA_REGISTRY_URL: http://schema-registry:8081
      GENERATION_INTERVAL: 30.0
      BATCH_SIZE: 5
    networks:
      - streaming-network

  # Kafka UI (pour monitoring)
  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: connect
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect:8083
    networks:
      - streaming-network
    depends_on:
      - kafka
      - schema-registry
      - kafka-connect

networks:
  streaming-network:
    driver: bridge

volumes:
  kafka-data:
  clickhouse-data:
  superset-data: