PySparkDockerExample/docker-compose.yml at master · willardmr/PySparkDockerExample · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
version: "2"
services:
  spark-master:
    image: bde2020/spark-master:2.4.5-hadoop2.7
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    environment:
      - ENABLE_INIT_DAEMON=false
  spark-worker-1:
    image: bde2020/spark-worker:2.4.5-hadoop2.7
    container_name: spark-worker-1
    depends_on:
      - spark-master
    ports:
      - "8082:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
  spark-worker-2:
    image: bde2020/spark-worker:2.4.5-hadoop2.7
    container_name: spark-worker-2
    depends_on:
      - spark-master
    ports:
      - "8083:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
  py-spark:
    build: .
    depends_on:
      - spark-master
    environment:
      - ENABLE_INIT_DAEMON=false
      - "SPARK_MASTER=spark://spark-master:7077"