docker-compose.yml

version: '3' 
services:
  postgres:
    image: fjardim/mds-postgres
    container_name: postgres
    hostname: postgres
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      POSTGRES_DB: admin 
    ports:
      - 5442:5432
    volumes:
      - ./postgres/postgres-db-volume:/var/lib/postgresql/data
      - ./postgres:/util
      - ./scripts:/scripts
    #healthcheck:
    #  test: ["CMD", "pg_isready"]
    #  interval: 5s
    #  retries: 5
  
  
  spark-master:
    image: fjardim/mds-spark
    hostname: spark-master
    container_name: spark-master
    command: 
      - /bin/sh
      - -c
      - |
        /usr/local/spark/sbin/start-master.sh
        start-notebook.sh --NotebookApp.token=''
    #environment:
    #  PYSPARK_SUBMIT_ARGS: "--packages io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.392"
    #env_file:
    #  - ./data/jupyter/jupyter.env
    #working_dir: /home/root
    #entrypoint: ["chmod", "+x", "/env/start-master.sh","/env","./start-master.sh"]
    ports:
      - 8889:8888
      - 4040:4040
      - 4041:4041
      - 4042:4042
      - 4043:4043
      - 8180:8080
      - 7077:7077
    volumes:
      - ./spark/util:/util/
      - ./spark/work:/home/user 
      - ./spark/env:/env 
      - ./scripts:/scripts
      #- ./jupyter/conf:/usr/local/spark/conf
    #environment:
    #  - GRANT_SUDO=yes
    #  - CHOWN_HOME=yes
    #  - NB_USER=spark
    #  - NB_GID=100
    #  - CHOWN_HOME_OPTS='-R'
      #- GEN_CERT=yes

    #   SPARK_MASTER: local[*]
    #   JUPYTER_PORT: 8889
    #deploy:
    #  resources:
    #    limits:
    #      memory: 500m

  spark-worker:
    image: fjardim/mds-spark
    hostname: spark-worker
    container_name: spark-worker
    #command: start-notebook.sh --NotebookApp.token='' 
    command: 
      - /bin/sh
      - -c
      - |
        /usr/local/spark/sbin/start-worker.sh spark-master:7077
        start-notebook.sh --NotebookApp.token='' 
    #command: /usr/local/spark/sbin/start-worker.sh jupyter-spark:7077
    #environment:
    #  PYSPARK_SUBMIT_ARGS: "--packages io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.392"
    env_file:
      - ./spark/env/jupyter.env
    #working_dir: /home/root
    ports:
      - 5040:4040
      - 5041:4041
      - 5042:4042
      - 5043:4043
      - 8881:8081
      - 36533:36533
    volumes:
      - ./spark/util:/util/
      - ./spark/work:/home/user 
      - ./scripts:/scripts
    environment:
      SPARK_MASTER: spark-master
    #   JUPYTER_PORT: 8889
    depends_on:
        - spark-master
    #deploy:
    #  resources:
    #    limits:
    #      memory: 1g

  trino:
    image: fjardim/mds-trino 
    #image: trinodb/trino:latest
    container_name: trino
    hostname: trino
    ports: 
      - "8080:8080" 
    depends_on:
      - postgres
      #- metastore
    volumes: 
      - ./trino/catalog:/etc/trino/catalog  
      - ./trino/conf:/conf
      - ./scripts:/scripts

  hive:
    hostname: hive
    #image: apache/hive:4.0.0-alpha-2
    image: fjardim/mds-hive
    container_name: hive
    environment:
      AWS_ACCESS_KEY_ID: datalake
      AWS_SECRET_ACCESS_KEY: datalake
      HIVE_CUSTOM_CONF_DIR: "/hive_custom_conf"
      SERVICE_NAME: hiveserver2
      SERVICE_OPTS: "-Dhive.metastore.uris=thrift://metastore:9083" 
      IS_RESUME: "true"
      #HIVE_VERSION: "3.1.3"
    ports:
       - "10000:10000"
       - "10002:10002"
    depends_on:
        - hive-metastore
    user: root
    volumes:
       - ./hive/conf:/hive_custom_conf
       - ./hive:/util

  hive-metastore:
    hostname: metastore
    image: fjardim/mds-hive-metastore
    #image: apache/hive:4.0.0-alpha-2
    container_name: metastore
    environment:
      AWS_ACCESS_KEY_ID: datalake
      AWS_SECRET_ACCESS_KEY: datalake
      HIVE_CUSTOM_CONF_DIR: "/hive_custom_conf"
      SERVICE_NAME: metastore
      #SERVICE_OPTS: "-Dhive.metastore.uris=thrift://metastore:9083" 
      IS_RESUME: "true"
      DB_DRIVER: postgres 
      SERVICE_OPTS: "-Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore_db -Djavax.jdo.option.ConnectionUserName=admin -Djavax.jdo.option.ConnectionPassword=admin" 
    ports:
       - "9083:9083"
    depends_on: 
      - postgres
    user: root
    volumes:
       - ./hive/meta:/opt/hive/data/warehouse 
       - ./hive/conf:/hive_custom_conf
       - ./hive:/util

  minio:
    image: fjardim/mds-minio
    container_name: minio
    command:  server --console-address ":9001" /data
    ports:
      - "9050:9000"
      - "9051:9001"
    hostname: minio
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: minioadmin
      MINIO_ACCESS_KEY: datalake
      MINIO_SECRET_KEY: datalake
    volumes:
      - ./minio/data1:/data
      - ./scripts:/scripts

  namenode:
    image: fjardim/mds-namenode
    container_name: namenode
    hostname: namenode
    volumes:
      - ./hadoop/hdfs/namenode:/hadoop/dfs/name
      - ./hadoop/util:/util
    env_file:
      - ./hadoop/hadoop.env
    ports:
      - "9870:9870"
    deploy:
      resources:
        limits:
          memory: 500m
  
  datanode:
    image: fjardim/mds-datanode
    container_name: datanode
    hostname: datanode
    volumes:
      - ./hadoop/hdfs/datanode:/hadoop/dfs/data
      - ./hadoop/util:/util
    env_file:
      - ./hadoop/hadoop.env
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    depends_on:
      - namenode
    ports:
      - "9864:9864"
    deploy:
      resources:
        limits:
          memory: 500m

  hue:
    image: fjardim/mds-hue
    hostname: hue
    container_name: hue
    dns: 8.8.8.8
    ports:
    - "8888:8888"
    volumes:
      - ./hue/hue.ini:/usr/share/hue/desktop/conf/z-hue.ini
    depends_on:
      - "postgres"
    deploy:
      resources:
        limits:
          memory: 500m