testing.yml

name: Automated Tests

on:
  push:
    branches: master
  pull_request:
    branches: master

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: 3.11
    - name: Install flake8
      shell: bash
      run: |
        python -V
        python -m pip install "flake8<6.0.0"
    # XXX: flake8 --diff is deprecated, broken and was removed
    # in flake8 6.0.0: we should instead black-ify the full repo
    # and run a full flake8 check at each PR.
    - name: Run flake8 on diff with upstream/master
      shell: bash
      run: |
        bash ./.github/scripts/flake8_diff.sh

  build:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9"]
        exclude:
          # Do not test all minor versions on all platforms, especially if they
          # are not the oldest/newest supported versions
          - os: windows-latest
            python_version: 3.8
            # as of  4/02/2020, psutil won't build under PyPy + Windows
          - os: windows-latest
            python_version: "pypy-3.9"
          - os: macos-latest
            python_version: 3.8
          - os: macos-latest
            # numpy triggers: RuntimeError: Polyfit sanity test emitted a
            # warning
            python_version: "pypy-3.9"

    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python_version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
        allow-prereleases: true
    - name: Install project and dependencies
      shell: bash
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e .
        python -m pip install -r dev-requirements.txt
        python ci/install_coverage_subprocess_pth.py
        export
    - name: Display Python version
      shell: bash
      run: python -c "import sys; print(sys.version)"
    - name: Look for syntax errors/undefined names
      shell: bash
      run: |
        python -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 \
          --show-source --statistics
    - name: Test with pytest
      shell: bash
      run: |
        COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
          PYTHONPATH='.:tests' python -m pytest -r s
        coverage combine --append
        coverage xml -i
    - name: Publish coverage results
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  distributed-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci distributed') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: distributed
      TEST_REQUIREMENTS: cryptography pytest pytest-asyncio<0.14.0 pytest-timeout pytest-rerunfailures numpy pandas mock bokeh fsspec>=0.3.3 aiohttp pyarrow git+https://github.com/dask/dask
      PROJECT_URL: https://github.com/dask/distributed.git
    strategy:
      matrix:
        python_version: ["3.10"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Install developed version of cloudpickle
      shell: bash
      run: |
        python -m pip install -e .
    - name: Test the downstream project
      shell: bash
      run: |
        # FIXME ipv6-related failures on Ubuntu github actions CI
        # https://github.com/dask/distributed/issues/4514
        export DISABLE_IPV6=1
        export PYTEST_ADDOPTS=("-m" "not avoid_ci")
        source ./.github/scripts/test_downstream_project.sh

  joblib-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci joblib') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: joblib
      TEST_REQUIREMENTS: "threadpoolctl pytest numpy distributed"
      PROJECT_URL: https://github.com/joblib/joblib.git
    strategy:
      matrix:
        python_version: ["3.10"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Vendor current cloudpickle inside joblib
      run: |
        pushd ../joblib/joblib/externals
        source vendor_cloudpickle.sh ../../../cloudpickle
        popd
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  loky-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci loky') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: loky
      TEST_REQUIREMENTS: "pytest psutil"
      PROJECT_URL: https://github.com/joblib/loky.git
    strategy:
      matrix:
        python_version: ["3.10"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install downstream project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Install developed version of cloudpickle
      shell: bash
      run: |
        python -m pip install -e .
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  ray-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci ray') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: ray
    strategy:
      matrix:
        python_version: ["3.10"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        python -m pip install --upgrade -r dev-requirements.txt
        python -m pip install setproctitle psutil
        # from https://docs.ray.io/en/master/development.html#building-ray
        pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl
        pushd ..
        git clone https://github.com/ray-project/ray.git
        pushd ray
        python python/ray/setup-dev.py --yes
        popd
        popd
        PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")
        rm $PROJECT_DIR/cloudpickle/cloudpickle.py
        cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py
        cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py
        cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py
    - name: Test the downstream project
      run: |
        PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")"
        COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_simple_serialization
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_complex_serialization
        pytest -vl $PROJECT_DIR/tests/test_basic.py::test_ray_recursive_objects
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_serialization_final_fallback
        pytest -vl $PROJECT_DIR/tests/test_basic.py::test_nested_functions