spark_complete_guide_1516/docker-compose.yaml at main · 05kim/spark_complete_guide_1516 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
version: "3"
services:
  spark-master:
    image: bde2020/spark-master:3.3.0-hadoop3.3
    container_name: spark-master
    working_dir: /spark
    volumes:
      - ./app:/spark/app
    ports:
      - "4040:4040"
      - "8080:8080"
      - "7077:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark
      - SPARK_WORKER_INSTANCES=10

  spark-worker-1:
    image: bde2020/spark-worker:3.3.0-hadoop3.3
    container_name: spark-worker-1
    working_dir: /spark
    depends_on:
      - spark-master
    volumes:
      - ./app:/spark/app
    ports:
      - "8081:8081"
      - "4041:4040"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - SPARK_WORKER_INSTANCES=10
  spark-worker-2:
    image: bde2020/spark-worker:3.3.0-hadoop3.3
    container_name: spark-worker-2
    working_dir: /spark
    volumes:
      - ./app:/spark/app
    depends_on:
      - spark-master
    ports:
      - "8082:8082"
      - "4042:4040"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - SPARK_WORKER_INSTANCES=10
  spark-worker-3:
    image: bde2020/spark-worker:3.3.0-hadoop3.3
    container_name: spark-worker-3
    working_dir: /spark
    volumes:
      - ./app:/spark/app
    depends_on:
      - spark-master
    ports:
      - "8083:8083"
      - "4043:4040"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - SPARK_WORKER_INSTANCES=10
  mysql:
    image: mysql:8.0
    container_name: mysql
    ports:
      - 3306:3306 # HOST:CONTAINER
    environment:
      MYSQL_ROOT_PASSWORD: admin
    command:
      - --character-set-server=utf8mb4
      - --collation-server=utf8mb4_unicode_ci
    volumes:
      - ./data:/var/lib/mysql
  mongodb:
    image: mongo
    container_name: mongodb
    restart: always
    ports:
      - 27017:27017
    volumes:
      - ./mongodb:/data/db
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=1234
      - MONGO_INITDB_DATABASE=test