Add Windows to Github Action CI matrix (openml#1095)

* Add Windows to Github Action CI matrix

* Fix syntax, disable Ubuntu tests

Ubuntu tests only temporarily disabled for this PR, to avoid
unnecessary computational costs/time.

* Fix syntax for skip on install Python step

* Explicitly add the OS to includes

* Disable check for files left behind for Windows

The check is bash script, which means it fails on a Windows machine.

* Re-enable Ubuntu tests

* Replace Appveyor with Github Actions for WindowsCI

Author: PGijsbers

.github/workflows/ubuntu-test.yml renamed to .github/workflows/test.yml

@@ -3,27 +3,34 @@ name: Tests
 on: [push, pull_request]
 
 jobs:
-  ubuntu:
-
-    runs-on: ubuntu-latest
+  test:
+    name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }})
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8]
         scikit-learn: [0.21.2, 0.22.2, 0.23.1, 0.24]
+        os: [ubuntu-latest]
         exclude:  # no scikit-learn 0.21.2 release for Python 3.8
           - python-version: 3.8
             scikit-learn: 0.21.2
         include:
           - python-version: 3.6
             scikit-learn: 0.18.2
             scipy: 1.2.0
+            os: ubuntu-latest
           - python-version: 3.6
             scikit-learn: 0.19.2
+            os: ubuntu-latest
           - python-version: 3.6
             scikit-learn: 0.20.2
+            os: ubuntu-latest
           - python-version: 3.8
             scikit-learn: 0.23.1
             code-cov: true
+            os: ubuntu-latest
+          - os: windows-latest
+            scikit-learn: 0.24.*
       fail-fast:  false
       max-parallel: 4
 
@@ -32,6 +39,7 @@ jobs:
       with:
         fetch-depth: 2
     - name: Setup Python ${{ matrix.python-version }}
+      if: matrix.os != 'windows-latest'  # windows-latest only uses preinstalled Python (3.7.9)
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
@@ -50,12 +58,17 @@ jobs:
       id: status-before
       run: |
         echo "::set-output name=BEFORE::$(git status --porcelain -b)"
-    - name: Run tests
+    - name: Run tests on Ubuntu
+      if: matrix.os == 'ubuntu-latest'
       run: |
         if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long  --cov-report=xml'; fi
         pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov --reruns 5 --reruns-delay 1
+    - name: Run tests on Windows
+      if: matrix.os == 'windows-latest'
+      run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
+        pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv --reruns 5 --reruns-delay 1
     - name: Check for files left behind by test
-      if: ${{ always() }}
+      if: matrix.os != 'windows-latest' && always()
       run: |
         before="${{ steps.status-before.outputs.BEFORE }}"
         after="$(git status --porcelain -b)"
@@ -71,4 +84,4 @@ jobs:
       with:
         files: coverage.xml
         fail_ci_if_error: true
-        verbose: true
+        verbose: true

appveyor.yml

@@ -6,6 +6,12 @@
 Changelog
 =========
 
+0.13.0
+~~~~~~
+
+ * MAIN#1088: Do CI for Windows on Github Actions instead of Appveyor.
+
+
 0.12.2
 ~~~~~~
 

appveyor/run_with_env.cmd

@@ -366,10 +366,7 @@ def _check_sample_evaluations(
                             evaluation = sample_evaluations[measure][rep][fold][sample]
                             self.assertIsInstance(evaluation, float)
                             if not (os.environ.get("CI_WINDOWS") or os.name == "nt"):
-                                # Either Appveyor is much faster than Travis
-                                # and/or measurements are not as accurate.
-                                # Either way, windows seems to get an eval-time
-                                # of 0 sometimes.
+                                # Windows seems to get an eval-time of 0 sometimes.
                                 self.assertGreater(evaluation, 0)
                             self.assertLess(evaluation, max_time_allowed)