spark-on-yarn-cluster/spark-client-with-tpcds-docker-compose.yml at master · mohsenasm/spark-on-yarn-cluster · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
version: "3.2"

services:
  tpc-ds:
    build: tpc-ds
    image: mohsenasm/spark-on-yarn-cluster:tpc-ds.9
    environment:
      data_path: "hdfs:///tpc-ds-files/data"
    command: /run.sh exit
    networks:
      - yarn_net
    volumes:
      - tpc-ds-files:/tpc-ds-files

  spark-client:
    build: spark-client
    image: mohsenasm/spark-on-yarn-cluster:spark_client.8
    environment:
      SERVICE_PRECONDITION: "resourcemanager:8088"
      SPARK_HOME: "/opt/spark"
      PYSPARK_PYTHON: "python3"
    ports:
      - 18080:18080
    command: exit
    networks:
      - yarn_net
    env_file:
      - ./hadoop.env
    volumes:
      - tpc-ds-files:/tpc-ds-files:ro
      - spark-history:/spark-history
      - spark-derby-system:/spark-derby-system

  resourcemanager:
    build: yarn/resource_manager
    image: mohsenasm/spark-on-yarn-cluster:resource_manager.6
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
    ports: # maybe cause security issue
      - 8088:8088
    expose:
      # - 8088
      - 8032
      - 8030
      - 8031
    networks:
      - yarn_net
    env_file:
      - ./hadoop.env

  nodemanager:
    build: yarn/node_manager
    image: mohsenasm/spark-on-yarn-cluster:node_manager.6
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    expose:
      - "1000-65535"
    networks:
      - yarn_net
    env_file:
      - ./hadoop.env

  namenode:
    build: yarn/name_node
    image: mohsenasm/spark-on-yarn-cluster:name_node.6
    ports:
      - 9870:9870
    expose:
      - 9000
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    networks:
      - yarn_net
    env_file:
      - ./hadoop.env

  datanode:
    build: yarn/data_node
    image: mohsenasm/spark-on-yarn-cluster:data_node.6
    expose:
      - 9864
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    networks:
      - yarn_net
    env_file:
      - ./hadoop.env

  historyserver:
    build: yarn/history_server
    image: mohsenasm/spark-on-yarn-cluster:history_server.6
    ports:
      - 8188:8188
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    networks:
      - yarn_net
    env_file:
      - ./hadoop.env

volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver:
  tpc-ds-files:
  spark-history:
  spark-derby-system:

networks:
  yarn_net: