docker-compose.yml

version: '3.4'

services:

  spark-master:
    container_name: spark-master
    image: spark
    build:
      context: ./services/spark
      dockerfile: Dockerfile
    command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
    hostname: spark-master
    environment:
      MASTER: spark://spark-master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    volumes:
      - ./services/spark/conf/master:/conf
      - ./services/spark/data:/tmp/data

  spark-worker:
    image: spark
    container_name: spark-worker
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    hostname: spark-worker
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - 8081:8081
    volumes:
      - ./services/spark/conf/worker:/conf
      - ./services/spark/data:/tmp/data
    links:
      - spark-master
    depends_on:
      - spark-master

  web:
    container_name: web
    image: web
    build:
      context: ./services/web
      dockerfile: Dockerfile
    volumes:
      - './services/web/project:/usr/src/app'
    ports:
      - 5009:5000
    command: python manage.py runserver -h 0.0.0.0
    environment:
      - APP_SETTINGS=server.config.DevelopmentConfig
    links:
      - spark-master
      - spark-worker
    depends_on:
      - redis
      - spark-master
      - spark-worker

  worker:
    image: web
    container_name: redis-worker
    environment:
      - APP_SETTINGS=server.config.DevelopmentConfig
    command: spark-submit --master spark://spark-master:7077 --class endpoint manage.py run_worker
    volumes:
      - './services/web/project:/usr/src/app'
    depends_on:
      - redis
      - web

  redis:
    image: redis:3.2.11