build.yml

name: build

on: [push, pull_request]

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.7, ] 

    steps:
    - uses: actions/checkout@v1
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        make venv
    - name: Lint python files with flake8
      run: |
        make flake8
    - uses: olafurpg/setup-scala@v10
      with:
        java-version: adopt@1.8
    - name: Install Apache Spark
      run: |
        # This command will install vanilla spark under ./spark-2.4.8-bin-hadoop2.7
        make install_spark
    - name: Test with pytest
      run: |
        export SPARK_HOME=$(pwd)/spark-2.4.8-bin-hadoop2.7
        export PYTHONPATH=${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.7-src.zip:${PYTHONPATH}
        export PATH=${PATH}:${SPARK_HOME}/bin:${SPARK_HOME}/sbin
        make test
    - name: Convert notebook to python script
      run: |
        make py
        mkdir -p ima/scripts/
        mv algorithm.py ima/scripts
    - uses: actions/upload-artifact@v2
      with:
        name: algorithm
        path: ima/scripts/algorithm.py