docker-compose-project.yml

version: '3.7'
services:

    postgres:                                           # create postgres container
        image: postgres:9.6
        container_name: postgres_container
        environment:
            - POSTGRES_USER=airflow
            - POSTGRES_PASSWORD=airflow
            - POSTGRES_DB=airflow

    airflow:                                            # create airflow container
        build: './airflow_docker'                       # construct the container along the Dockerfile in this folder
        container_name: airflow_container
        restart: always
        depends_on:
            - postgres
        environment:
            - LOAD_EX=n
            - EXECUTOR=Local
        volumes:                                        # mount the following local folders
            - ./dags:/usr/local/airflow/dags
            - ./data:/usr/local/airflow/data
            - ./models:/usr/local/airflow/models
        ports:
            - "8080:8080"                               # expose port
        command: webserver
        healthcheck:
            test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
            interval: 30s
            timeout: 30s
            retries: 3

    zookeeper:                                          # create zookeeper container
        image: wurstmeister/zookeeper
        container_name: zookeeper_container
        ports:
            - "2181:2181"                               # expose port

    kafka:                                              # create an instance of a Kafka broker in a container
        image: wurstmeister/kafka
        container_name: kafka_container
        ports:
            - "9092:9092"                               # expose port
        environment:
            KAFKA_ADVERTISED_HOST_NAME: kafka           # specify the docker host IP at which other containers can reach the broker
            KAFKA_CREATE_TOPICS: "TopicA:1:1"           # create a topic called 'TopicA"  with 1 partition and 1 replica
            KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181     # specify where the broker can reach Zookeeper
            KAFKA_LISTENERS: PLAINTEXT://:9092          # the list of addresses on which the Kafka broker will listen on for incoming connections.
            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092  # Kafka sends the value of this variable to clients during their connection. After receiving that value, the clients use it for sending/consuming records to/from the Kafka broker.y connect to it.
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock
    mlflow:                                             # create a MLFlow container
        build: './mlflow_docker'                        # construct the container along the Dockerfile in this folder
        container_name: mlflow_container
        ports:
            - "5000:5000"                               # expose port
        command: 'mlflow server --backend-store-uri ./mlflow --host 0.0.0.0 --port 5000'