diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ab60f59c6..241455753 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,116 +1,138 @@
 name: Tests
-
 on:
   workflow_dispatch:
-
   push:
     branches:
       - main
       - develop
     tags:
       - "v*.*.*"
-
   pull_request:
     branches:
       - main
       - develop
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
-
+env:
+  cores: 4
+  durations: 20
+  timeout: 600
+  timeout_method: thread
+  dist: load
+  reruns: 5
+  reruns_delay: 1
+  log_cli: true
+  capture: no
 jobs:
   test:
     name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }})
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.8"]
-        # TODO(eddiebergman): We should consider testing against newer version I guess...
-        # We probably consider just having a `"1"` version to always test against latest
-        scikit-learn: ["0.23.1", "0.24"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-latest]
-        sklearn-only: ["true"]
-        exclude:  # no scikit-learn 0.23 release for Python 3.9
-          - python-version: "3.9"
-            scikit-learn: "0.23.1"
         include:
+          # TODO(eddiebergman): For Python 3.8, we can run sklearn {0.23.1, 0.24}
+          - os: ubuntu-latest
+            python-version: "3.8"
+            scikit-learn: "0.23.1"
+            numpy-version: "1.23.5" # TODO(eddiebergman): Legacy w/ sklearn
+            sklearn-only: true
+          - os: ubuntu-latest
+            python-version: "3.8"
+            scikit-learn: "0.24"
+            sklearn-only: true
+          # TODO(eddiebergman): For Python 3.8, we can run sklearn {0.24}
           - os: ubuntu-latest
             python-version: "3.9"
             scikit-learn: "0.24"
             scipy: "1.10.0"
-            sklearn-only: "true"
+            sklearn-only: true
           # Include a code cov version
           - code-cov: true
             os: ubuntu-latest
             python-version: "3.8"
             scikit-learn: 0.23.1
-            sklearn-only: 'false'
-          # Include a windows test, for some reason on a later version of scikit-learn
+          # TODO(eddiebergman): Do we really need a dedicated windows version?
+          # ... probably yes while we're not convinced we're using Path everywhere.
+          # Include a windows test
           - os: windows-latest
             python-version: "3.8"
             scikit-learn: 0.24.*
-            scipy: "1.10.0"  # not sure why the explicit scipy version?
-            sklearn-only: 'false'
-      fail-fast:  false
-
+            scipy: "1.10.0" # Fails with earlier version of scipy
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 2
-    - name: Setup Python ${{ matrix.python-version }}
-      if: matrix.os != 'windows-latest'  # windows-latest only uses preinstalled Python (3.7.9)
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install test dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .[test]
-    - name: Install scikit-learn ${{ matrix.scikit-learn }}
-      run: |
-        pip install scikit-learn==${{ matrix.scikit-learn }}
-    - name: Install numpy for Python 3.8
-      # Python 3.8 & scikit-learn<0.24 requires numpy<=1.23.5
-      if: ${{ matrix.python-version == '3.8' && contains(fromJSON('["0.23.1", "0.22.2", "0.21.2"]'), matrix.scikit-learn) }}
-      run: |
-        pip install numpy==1.23.5
-    - name: Install scipy ${{ matrix.scipy }}
-      if: ${{ matrix.scipy }}
-      run: |
-        pip install scipy==${{ matrix.scipy }}
-    - name: Store repository status
-      id: status-before
-      run: |
-        echo "::set-output name=BEFORE::$(git status --porcelain -b)"
-    - name: Run tests on Ubuntu
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long  --cov-report=xml'; fi
-        # Most of the time, running only the scikit-learn tests is sufficient
-        if [ ${{ matrix.sklearn-only }} = 'true' ]; then sklearn='-m sklearn'; fi
-        echo pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 -o log_cli=true
-        pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 -o log_cli=true
-    - name: Run tests on Windows
-      if: matrix.os == 'windows-latest'
-      run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
-        pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv --reruns 5 --reruns-delay 1
-    - name: Check for files left behind by test
-      if: matrix.os != 'windows-latest' && always()
-      run: |
-        before="${{ steps.status-before.outputs.BEFORE }}"
-        after="$(git status --porcelain -b)"
-        if [[ "$before" != "$after" ]]; then
-            echo "git status from before: $before"
-            echo "git status from after: $after"
-            echo "Not all generated files have been deleted!"
-            exit 1
-        fi
-    - name: Upload coverage
-      if: matrix.code-cov && always()
-      uses: codecov/codecov-action@v3
-      with:
-        files: coverage.xml
-        token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
-        verbose: true
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install test dependencies
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[test]"
+
+          if [ ${{ matrix.numpy-version }} ]; then
+              pip install numpy==${{ matrix.numpy-version }}
+          fi
+          if [ ${{ matrix.scikit-learn }} ]; then
+              pip install scikit-learn==${{ matrix.scikit-learn }}
+          else
+              pip install scikit-learn
+          fi
+          if [ ${{ matrix.scipy }} ]; then
+              pip install scipy==${{ matrix.scipy }}
+          else
+              pip install scipy
+          fi
+      - name: Store repository status
+        id: status_before
+        shell: bash
+        run: |
+          status="$(git status --porcelain -b)"
+          echo $status
+          echo "BEFORE=${status}" >> $GITHUB_ENV
+      - name: Run tests on Ubuntu
+        shell: bash
+        run: |
+          set -x  # Print out the command run
+          # Most of the time, running only the scikit-learn tests is sufficient
+          if [ ${{ matrix.sklearn-only }} ]; then sklearn="-m sklearn"; fi
+          if [ ${{ matrix.code-cov }} ]; then codecov="--cov=openml --long  --cov-report=xml"; fi
+          pytest \
+            --verbose \
+            -n "${{env.cores}}" \
+            --capture="${{env.capture}}" \
+            --durations="${{env.durations}}" \
+            --timeout="${{env.timeout}}" \
+            --timeout-method="${{env.timeout_method}}" \
+            --dist "${{env.dist}}" \
+            --reruns "${{env.reruns}}" \
+            --reruns-delay "${{env.reruns_delay}}" \
+            ${sklearn} \
+            ${codecov} \
+            -o log_cli="${{env.log_cli}}"
+      - name: Check for files left behind by test
+        if: always()
+        shell: bash
+        run: |
+          after="$(git status --porcelain -b)"
+          before="${{ env.BEFORE }}"
+          if [[ "${before}" != "${after}" ]]; then
+              echo "git status from before: ${before}"
+              echo "git status from after: ${after}"
+              echo "Not all generated files have been deleted!"
+              exit 1
+          fi
+      - name: Upload coverage
+        if: matrix.code-cov && always()
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+          verbose: true
diff --git a/pyproject.toml b/pyproject.toml
index 99ff2b804..68fe575c3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,8 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 license = { file = "LICENSE" }