docker-compose.yaml

version: '3.8'

services:

  # ======================= HDFS NAMENODE =======================
  # Stores HDFS metadata (directory tree, block locations)
  # Web UI: http://localhost:9870
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    ports:
      - "9870:9870"   # Web UI — check health here after boot
      - "9000:9000"   # Legacy port (some tools use this)
    environment:
      - CLUSTER_NAME=ids-cluster
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
      - HDFS_CONF_dfs_replication=1   # Only 1 datanode, replication must be 1
    volumes:
      - ./data/hdfs-namenode:/hadoop/dfs/name   # Bind mount — survives compose down
    networks:
      - ids-net
    deploy:
      resources:
        limits:
          memory: 768m
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9870"]
      interval: 30s
      timeout: 10s
      retries: 5

  # ======================= HDFS DATANODE =======================
  # Actually stores your dataset blocks on disk
  # Web UI: http://localhost:9864
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    ports:
      - "9864:9864"
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data
      - HDFS_CONF_dfs_replication=1
    volumes:
      - ./data/hdfs-datanode:/hadoop/dfs/data   # Bind mount — actual data lives here
    networks:
      - ids-net
    depends_on:
      namenode:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 768m
    restart: unless-stopped

  # ======================= SPARK MASTER =======================
  # Standalone Spark master — coordinates workers, no YARN needed
  # Web UI: http://localhost:8080
  # Job UI: http://localhost:4040 (only visible while a job runs)
  spark:
    image: apache/spark:3.5.1
    container_name: spark
    command: /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master
    ports:
      - "8080:8080"   # Spark master Web UI
      - "7077:7077"   # Workers and driver connect here
      - "4040:4040"   # Job UI
    environment:
      - SPARK_MASTER_HOST=spark
      - SPARK_MASTER_WEBUI_PORT=8080
      - SPARK_DAEMON_MEMORY=512m
    networks:
      - ids-net
    depends_on:
      namenode:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 768m
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080"]
      interval: 30s
      timeout: 10s
      retries: 5

  # ======================= SPARK WORKER =======================
  # Does the actual computation. Only 1 worker to stay within RAM budget.
  # To add a second worker, uncomment spark-worker-2 at the bottom
  # ONLY after confirming this setup is stable (check: docker stats)
  spark-worker-1:
    image: apache/spark:3.5.1
    container_name: spark-worker-1
    command: /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark:7077
    environment:
      - SPARK_WORKER_MEMORY=1536m   # What worker advertises to master
      - SPARK_WORKER_CORES=2
      - SPARK_EXECUTOR_MEMORY=1g    # Executor heap — leaves headroom within 1536m
      - SPARK_EXECUTOR_CORES=1
      # Cleanup old app dirs, and tune GC for low-memory
      - SPARK_WORKER_OPTS=-Dspark.worker.cleanup.enabled=true -Dspark.worker.cleanup.interval=300
    networks:
      - ids-net
    depends_on:
      spark:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 2g    # Hard Docker cap — this is what prevents exit code 52 OOM kills
    restart: unless-stopped

  # ======================= JUPYTER (CUSTOM IMAGE) =======================
  # Your main dev environment. Build it first:
  #   docker build -f Dockerfile.jupyter -t spark-jupyter:3.5.1 .
  # Access: http://localhost:8888 (no password)
  spark-jupyter:
    image: spark-jupyter:3.5.1
    container_name: spark-jupyter
    ports:
      - "8888:8888"
    volumes:
      - ./workspace:/opt/work    # Notebooks, scripts, models all persist here
    environment:
      - SPARK_MASTER=spark://spark:7077
      - JAVA_HOME=/opt/java/openjdk
      - PYSPARK_SUBMIT_ARGS=--master spark://spark:7077
          --driver-memory 1g
          --executor-memory 1g
          --executor-cores 1
          --conf spark.sql.shuffle.partitions=8
          --conf spark.network.timeout=300s
          --conf spark.executor.heartbeatInterval=60s
          pyspark-shell
      - MALLOC_ARENA_MAX=2
    command: >
      jupyter lab --ip=0.0.0.0 --port=8888 --no-browser
      --allow-root --NotebookApp.token='' --NotebookApp.password=''
    networks:
      - ids-net
    depends_on:
      spark:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 1536m
    restart: unless-stopped

  # ======================= KAFKA (KRaft mode) =======================
  # No Zookeeper needed — Kafka 7.5+ supports KRaft (built-in consensus)
  # Internal access (container-to-container): kafka:9092
  # External access (from your Mac): localhost:9093
  kafka:
    image: confluentinc/cp-kafka:7.5.0
    container_name: kafka
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_LOG_RETENTION_HOURS: 24           # Only keep 24h — saves disk
      KAFKA_LOG_SEGMENT_BYTES: 536870912      # 512MB segment cap
      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk     # Required for KRaft — fixed base64 string
      KAFKA_HEAP_OPTS: -Xmx512m -Xms256m     # Cap Kafka JVM heap
    volumes:
      - ./data/kafka:/var/lib/kafka/data
    networks:
      - ids-net
    deploy:
      resources:
        limits:
          memory: 768m
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "kafka-topics", "--bootstrap-server", "kafka:9092", "--list"]
      interval: 30s
      timeout: 10s
      retries: 10
      start_period: 30s    # Kafka takes ~20s to start in KRaft mode

  # ======================= MONGODB =======================
  # Stores prediction results, detection logs
  # Access: mongodb://localhost:27017
  mongodb:
    image: mongo:6
    container_name: mongodb
    ports:
      - "27017:27017"
    volumes:
      - ./data/mongodb:/data/db
    networks:
      - ids-net
    deploy:
      resources:
        limits:
          memory: 384m
    restart: unless-stopped

  # ======================= OPTIONAL: SECOND WORKER =======================
  # Uncomment ONLY if docker stats shows consistent headroom
  # and spark-worker-1 is stable for a while first.
  #
  # spark-worker-2:
  #   image: apache/spark:3.5.1
  #   container_name: spark-worker-2
  #   command: /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark:7077
  #   environment:
  #     - SPARK_WORKER_MEMORY=1536m
  #     - SPARK_WORKER_CORES=2
  #     - SPARK_EXECUTOR_MEMORY=1g
  #     - SPARK_EXECUTOR_CORES=1
  #   networks:
  #     - ids-net
  #   depends_on:
  #     spark:
  #       condition: service_healthy
  #   deploy:
  #     resources:
  #       limits:
  #         memory: 2g
  #   restart: unless-stopped

networks:
  ids-net:
    driver: bridge
    name: ids-net

# ================================================================
# NO named volumes defined — all storage is bind-mounted to ./data/
# This means docker compose down NEVER deletes your data.
# Only manually running `rm -rf ./data/` would delete it.
#
# FIRST-TIME SETUP — run this before docker compose up:
#
#   mkdir -p data/hdfs-namenode data/hdfs-datanode data/kafka data/mongodb
#   mkdir -p workspace/notebooks workspace/scripts workspace/models
#   mkdir -p workspace/dataset workspace/output workspace/logs
#   chmod -R 777 data/ workspace/
#
# TO START:    docker compose up -d
# TO STOP:     docker compose down          <-- safe, data persists
# TO DESTROY:  docker compose down -v       <-- still safe (no named vols)
#              rm -rf ./data/               <-- THIS deletes everything
# ================================================================