From 8632a9d2bb5d2b47712473b002934f2fd132c568 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 22:42:19 +0530
Subject: [PATCH 01/21] Test against NumPy nightlies

---
 .github/workflows/test.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cc95b1cf..5b7e78d7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,8 +21,7 @@ concurrency:
 
 jobs:
   test:
-    # name: Test / ${{ matrix.platform }} / Nightly ${{ matrix.nightly[0] }} / Python ${{ matrix.python-version }}
-    name: Test / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
+    name: Test / ${{ matrix.platform }} / Nightly ${{ matrix.nightly[0] }} / Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
@@ -30,8 +29,7 @@ jobs:
         platform: [ubuntu-latest, macos-13, macos-latest, windows-latest]
         python-version:
           ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9", "pypy-3.10"]
-        # TODO: disable nightly NumPy tests for now, re-enable later
-        # nightly: [[True, "nightly-"], [False, ""]]
+        nightly: [[True, "nightly-"], [False, ""]]
     steps:
       - uses: actions/checkout@v4.1.7
       - uses: actions/setup-python@v5.1.1
@@ -41,10 +39,8 @@ jobs:
 
       - name: Run CPython tests
         if: ${{ !startsWith(matrix.python-version, 'pypy') }}
-        # run: uvx nox -s ${{ matrix.nightly[1] }}tests
         run: uvx nox -s tests
 
       - name: Run PyPy tests
         if: ${{ startsWith(matrix.python-version, 'pypy') }}
-        # run: uvx nox -s ${{ matrix.nightly[1] }}tests
-        run: uvx nox -s tests
+        run: uvx nox -s ${{ matrix.nightly[1] }}tests

From 1b0c3da26dc14ac47ddefc0b3008246b11ea3ff7 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 15 Oct 2024 21:59:55 +0530
Subject: [PATCH 02/21] Split into a nightly test job, rework the workflow

---
 .github/workflows/test.yml | 50 ++++++++++++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5b7e78d7..020c7a44 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,15 +21,23 @@ concurrency:
 
 jobs:
   test:
-    name: Test / ${{ matrix.platform }} / Nightly ${{ matrix.nightly[0] }} / Python ${{ matrix.python-version }}
+    name: Regular tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-13, macos-latest, windows-latest]
         python-version:
-          ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9", "pypy-3.10"]
-        nightly: [[True, "nightly-"], [False, ""]]
+          [
+            "3.8",
+            "3.9",
+            "3.10",
+            "3.11",
+            "3.12",
+            "3.13",
+            "pypy-3.9",
+            "pypy-3.10",
+          ]
     steps:
       - uses: actions/checkout@v4.1.7
       - uses: actions/setup-python@v5.1.1
@@ -37,10 +45,36 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - uses: yezz123/setup-uv@v4
 
-      - name: Run CPython tests
-        if: ${{ !startsWith(matrix.python-version, 'pypy') }}
+      # On PyPy, we skip SciPy because we don't have wheels
+      # available, see noxfile.py for more details.
+      - name: Run tests
         run: uvx nox -s tests
 
-      - name: Run PyPy tests
-        if: ${{ startsWith(matrix.python-version, 'pypy') }}
-        run: uvx nox -s ${{ matrix.nightly[1] }}tests
+  # In this job, we test against the NumPy nightly wheels hosted on
+  # https://anaconda.org/scientific-python-nightly-wheels/numpy
+  # on the latest Python version available across platforms, instead of
+  # testing all Python versions and implementations on all platforms.
+  # We do not test on PyPy.
+  #
+  # However, "nox -s nightly-tests" can be used locally anywhere, on
+  # any Python version and implementation on any platform and we leave
+  # it to the user to decide what Python version to test against, which
+  # might or might not have a corresponding NumPy nightly wheel present.
+  nightlies:
+    name: Nightly tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, macos-13, macos-latest, windows-latest]
+        python-version: ["3.x"]
+
+    steps:
+      - uses: actions/checkout@v4.1.7
+      - uses: actions/setup-python@v5.1.1
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+      - uses: yezz123/setup-uv@v4
+      - name: Run tests against nightly wheels for NumPy and SciPy
+        run: uvx nox -s nightly-tests

From b48ebd05c24626edcd02856ab0c89192e7386851 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 1 Dec 2024 19:30:40 +0530
Subject: [PATCH 03/21] Fix type demotion issue in `combo_check`

---
 tests/test_scipy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_scipy.py b/tests/test_scipy.py
index 3662883a..ba563eff 100644
--- a/tests/test_scipy.py
+++ b/tests/test_scipy.py
@@ -228,7 +228,7 @@ def test_dirichlet_logpdf_alpha():
     ### Misc ###
     def test_logsumexp1():
         combo_check(special.logsumexp, [0], modes=["fwd", "rev"])(
-            [1.1, R(4), R(3, 4)], axis=[None, 0], keepdims=[True, False]
+            [np.array([1.1]), R(4), R(3, 4)], axis=[None, 0], keepdims=[True, False]
         )
 
     def test_logsumexp2():

From e384483fa42baabd0fd4dd785bd7a95fb8e05673 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 1 Dec 2024 19:42:29 +0530
Subject: [PATCH 04/21] Enforce binary wheels are downloaded from SPNW

---
 noxfile.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/noxfile.py b/noxfile.py
index e19f9d80..56b2e132 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -50,7 +50,11 @@ def run_nightly_tests(session):
     session.install("-e", ".[test]", silent=False)
     # SciPy doesn't have wheels on PyPy
     if platform.python_implementation() == "PyPy":
-        session.install("numpy", "--upgrade", silent=False, env=UV_NIGHTLY_ENV_VARS)
+        session.install(
+            "numpy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
+        )
     else:
-        session.install("numpy", "scipy", "--upgrade", silent=False, env=UV_NIGHTLY_ENV_VARS)
+        session.install(
+            "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
+        )
     session.run("pytest", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs)

From c6b8a41abcbfe65d16940a6c5609499d90d6df85 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Mon, 2 Dec 2024 06:39:43 +0530
Subject: [PATCH 05/21] Add a free-threading nox session

---
 noxfile.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/noxfile.py b/noxfile.py
index 56b2e132..8f6619c4 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -47,6 +47,7 @@ def ruff(session):
 @nox.session(name="nightly-tests", tags=["tests"])
 def run_nightly_tests(session):
     """Run tests against nightly versions of dependencies"""
+    session.run("python", "-VV")
     session.install("-e", ".[test]", silent=False)
     # SciPy doesn't have wheels on PyPy
     if platform.python_implementation() == "PyPy":
@@ -58,3 +59,28 @@ def run_nightly_tests(session):
             "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
         )
     session.run("pytest", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs)
+
+
+# Wheels for NumPy and SciPy are available as nightly builds, so we test
+# against them on Python 3.13t, which is the only version that supports
+# free-threaded Python.
+@nox.session(name="free-threading", python=["3.13t"])
+def run_with_free_threaded_python(session):
+    """Run tests with free threaded Python (no-GIL)"""
+    session.run("python", "-VV")
+    session.install("-e", ".[test]", silent=False)
+
+    # Add PYTHON_GIL=0 to UV_NIGHTLY_ENV_VARS
+    uv_nightly_env_vars_free_threading = UV_NIGHTLY_ENV_VARS.copy()
+    uv_nightly_env_vars_free_threading["PYTHON_GIL"] = "0"
+
+    # SciPy doesn't have wheels on PyPy
+    if platform.python_implementation() == "PyPy":
+        session.install(
+            "numpy", "--upgrade", "--only-binary", ":all:", silent=False, env=uv_nightly_env_vars_free_threading
+        )
+    else:
+        session.install(
+            "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=uv_nightly_env_vars_free_threading
+        )
+    session.run("pytest", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs)

From 79124bd2e7f8d29e10ce7f37799f11b0aa516450 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Mon, 2 Dec 2024 06:47:36 +0530
Subject: [PATCH 06/21] Add a noGIL job with forked `setup-python`

---
 .github/workflows/test.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ac6c4758..65c557d2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -72,3 +72,29 @@ jobs:
       - uses: yezz123/setup-uv@v4
       - name: Run tests against nightly wheels for NumPy and SciPy
         run: uvx nox -s nightly-tests
+
+
+  # In this job, we test with the free-threaded (no-GIL) Python 3.13t
+  # interpreter on all platforms. This is a special job and uses the
+  # "free-threading" session in noxfile.py. This job is available only
+  # for Python 3.13t at the time of writing and uses the nightly wheels
+  # for NumPy and SciPy, since free-threaded builds for either of them
+  # are not available on PyPI yet.
+  free-threaded-testing:
+    name: Free-threading tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, macos-13, macos-latest, windows-latest]
+        python-version: ["3.13t"]
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - uses: Quansight-Labs/setup-python@b9ab292c751a42bcd2bb465b7fa202ea2c3f5796 # v5.3.1
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+
+      - uses: yezz123/setup-uv@v4
+      - name: Run free-threaded tests on ${{ matrix.platform}}
+        run: uvx nox -s free-threading

From 331d9b80579a947c71538d5fefdd5b2110d497da Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Mon, 2 Dec 2024 06:47:50 +0530
Subject: [PATCH 07/21] Document `free-threading` `nox` session

---
 CONTRIBUTING.md | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7b87093c..77043b21 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -10,14 +10,15 @@ pip install nox
 
 ## Run tests, linting, packaging checks
 
-| Command                   | Description                                                                                                                                                     |
-| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `nox --list`              | Lists all available Nox sessions, including selected ones                                                                                                       |
-| `nox -s lint`             | Runs code style checks with pre-commit and pre-commit hooks as listed in `.pre-commit-config.yaml`. Accepts posargs to pass additional arguments to the linter. |
-| `nox -s tests`            | Runs tests with your default Python interpreter. Accepts posargs to pass additional arguments and configuration to `pytest`.                                    |
-| `nox -s nightly-tests`    | Similar to `nox -s tests`, except that it runs tests with nightly versions of dependencies (NumPy, SciPy, etc.).                                                |
-| `nox -s validate-package` | Builds a source distribution and a wheel using `pypa/build` and checks the package with `twine` in strict mode.                                                 |
-| `nox`                     | Runs all selected sessions, as listed in `nox.options.sessions` in `noxfile.py`.                                                                                |
+| Command                   | Description                                                                                                                                                          |
+| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `nox --list`              | Lists all available Nox sessions, including selected ones                                                                                                            |
+| `nox -s lint`             | Runs code style checks with pre-commit and pre-commit hooks as listed in `.pre-commit-config.yaml`. Accepts posargs to pass additional arguments to the linter.      |
+| `nox -s tests`            | Runs tests with your default Python interpreter. Accepts posargs to pass additional arguments and configuration to `pytest`.                                         |
+| `nox -s nightly-tests`    | Similar to `nox -s tests`, except that it runs tests with nightly versions of dependencies (NumPy, SciPy, etc.).                                                     |
+| `nox -s free-threading`   | Similar to `nox -s nightly-tests`, except that it runs tests with a free-threaded Python interpreter and corresponding builds for dependencies (NumPy, SciPy, etc.). |
+| `nox -s validate-package` | Builds a source distribution and a wheel using `pypa/build` and checks the package with `twine` in strict mode.                                                      |
+| `nox`                     | Runs all selected sessions, as listed in `nox.options.sessions` in `noxfile.py`.                                                                                     |
 
 Additionally, `nox` supports tags to run specific sessions, e.g., `nox --tags tests` runs all sessions tagged with `tests`.
 

From 6fb016bc75f00af156f2604e57a63e60f59d79c0 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Mon, 2 Dec 2024 06:48:28 +0530
Subject: [PATCH 08/21] Set `PYTHON_GIL` to `0` for the test suite

---
 noxfile.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/noxfile.py b/noxfile.py
index 8f6619c4..39db6d8d 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -63,24 +63,31 @@ def run_nightly_tests(session):
 
 # Wheels for NumPy and SciPy are available as nightly builds, so we test
 # against them on Python 3.13t, which is the only version that supports
-# free-threaded Python.
+# free-threaded Python. This session is similar to the "nightly-tests"
+# session, but it uses a free-threaded Python interpreter.
 @nox.session(name="free-threading", python=["3.13t"])
 def run_with_free_threaded_python(session):
     """Run tests with free threaded Python (no-GIL)"""
     session.run("python", "-VV")
     session.install("-e", ".[test]", silent=False)
 
-    # Add PYTHON_GIL=0 to UV_NIGHTLY_ENV_VARS
-    uv_nightly_env_vars_free_threading = UV_NIGHTLY_ENV_VARS.copy()
-    uv_nightly_env_vars_free_threading["PYTHON_GIL"] = "0"
-
     # SciPy doesn't have wheels on PyPy
     if platform.python_implementation() == "PyPy":
         session.install(
-            "numpy", "--upgrade", "--only-binary", ":all:", silent=False, env=uv_nightly_env_vars_free_threading
+            "numpy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
         )
     else:
         session.install(
-            "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=uv_nightly_env_vars_free_threading
+            "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
         )
-    session.run("pytest", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs)
+    # The PYTHON_GIL environment variable is set to 0, which enforces that
+    # extension modules that haven't declared themselves as safe to not rely
+    # on the GIL are run with the GIL disabled.
+    session.run(
+        "pytest",
+        "--cov=autograd",
+        "--cov-report=xml",
+        "--cov-append",
+        *session.posargs,
+        env={"PYTHON_GIL": "0"},
+    )

From a381e47017b39915bbf41670d26c66f27fe0256f Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Mon, 2 Dec 2024 06:49:27 +0530
Subject: [PATCH 09/21] Explicitly disable GIL fallback for workflow

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 65c557d2..baa9a600 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -83,6 +83,8 @@ jobs:
   free-threaded-testing:
     name: Free-threading tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.platform }}
+    env:
+      PYTHON_GIL: "0"
     strategy:
       fail-fast: false
       matrix:

From 98bd2dc36b6156eadff71aee507b50306659a524 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Mon, 2 Dec 2024 07:08:21 +0530
Subject: [PATCH 10/21] For now, don't set PYTHON_GIL env var in GHA job

Hoping that this fixes the issue when
installing Python 3.13t on macOS
---
 .github/workflows/test.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index baa9a600..a4f081b6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -83,8 +83,10 @@ jobs:
   free-threaded-testing:
     name: Free-threading tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.platform }}
-    env:
-      PYTHON_GIL: "0"
+    # Temporarily disable this environment variable to see if this fixes the message as follows:
+    # "Fatal Python error: config_read_gil: Disabling the GIL is not supported by this build"
+    # env:
+    #   PYTHON_GIL: "0"
     strategy:
       fail-fast: false
       matrix:

From 119d8ee59af67bce74bfeb65e5a7cdebe94c368a Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sat, 14 Dec 2024 20:31:48 +0530
Subject: [PATCH 11/21] Don't use coverage plugin or xdist for free-threaded
 tests

---
 noxfile.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/noxfile.py b/noxfile.py
index 39db6d8d..39ee9d2e 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -34,7 +34,9 @@ def run_tests(session):
         session.install("-e", ".[test]", silent=False)
     else:
         session.install("-e", ".[test,scipy]", silent=False)
-    session.run("pytest", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs)
+    session.run(
+        "pytest", "-n", "auto" "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs
+    )
 
 
 @nox.session(name="lint", reuse_venv=True)
@@ -58,18 +60,22 @@ def run_nightly_tests(session):
         session.install(
             "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
         )
-    session.run("pytest", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs)
+    session.run(
+        "pytest", "-n", "auto", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs
+    )
 
 
 # Wheels for NumPy and SciPy are available as nightly builds, so we test
 # against them on Python 3.13t, which is the only version that supports
 # free-threaded Python. This session is similar to the "nightly-tests"
-# session, but it uses a free-threaded Python interpreter.
+# session, but it uses a free-threaded Python interpreter. Also, we don't
+# the "test" extra but install the test dependencies manually.
 @nox.session(name="free-threading", python=["3.13t"])
 def run_with_free_threaded_python(session):
     """Run tests with free threaded Python (no-GIL)"""
     session.run("python", "-VV")
-    session.install("-e", ".[test]", silent=False)
+    session.install("-e", ".", silent=False)
+    session.install("pytest", silent=False)
 
     # SciPy doesn't have wheels on PyPy
     if platform.python_implementation() == "PyPy":
@@ -85,9 +91,6 @@ def run_with_free_threaded_python(session):
     # on the GIL are run with the GIL disabled.
     session.run(
         "pytest",
-        "--cov=autograd",
-        "--cov-report=xml",
-        "--cov-append",
         *session.posargs,
         env={"PYTHON_GIL": "0"},
     )

From b2a246f79da51ca92952913e21a9712cc77080d3 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sat, 14 Dec 2024 20:32:46 +0530
Subject: [PATCH 12/21] Remove required plugins, migrate additional options

---
 pyproject.toml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 2082f254..dc1bcbef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,10 +67,9 @@ source = ["autograd"]
 [tool.coverage.report]
 show_missing = true
 
-[tool.pytest.ini_options]
-required_plugins = ["pytest-cov", "pytest-xdist"]
 # TODO: generate HTML report, upload to CodeCov
-addopts = "--color=yes -sra -n auto --cov=autograd --cov-report=xml --cov-report=term"
+[tool.pytest.ini_options]
+addopts = "--color=yes -sra"
 
 [tool.ruff]
 extend-exclude = []

From 7c0ca367d875fe010c6a4ba8d4410e1e9af5e0d9 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sat, 14 Dec 2024 22:01:52 +0530
Subject: [PATCH 13/21] Add more free-threaded testing via plugins

---
 .github/workflows/test.yml | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a4f081b6..d7828699 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -81,7 +81,7 @@ jobs:
   # for NumPy and SciPy, since free-threaded builds for either of them
   # are not available on PyPI yet.
   free-threaded-testing:
-    name: Free-threading tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }}
+    name: "Free-threading tests / ${{ matrix.platform }} / Python ${{ matrix.python-version }} / Plugin ${{ matrix.plugin }}"
     runs-on: ${{ matrix.platform }}
     # Temporarily disable this environment variable to see if this fixes the message as follows:
     # "Fatal Python error: config_read_gil: Disabling the GIL is not supported by this build"
@@ -92,6 +92,7 @@ jobs:
       matrix:
         platform: [ubuntu-latest, macos-13, macos-latest, windows-latest]
         python-version: ["3.13t"]
+        plugin: ["none", "pytest-run-parallel", "pytest-freethreaded"]
     steps:
       - uses: actions/checkout@v4.2.2
       - uses: Quansight-Labs/setup-python@b9ab292c751a42bcd2bb465b7fa202ea2c3f5796 # v5.3.1
@@ -100,5 +101,27 @@ jobs:
           allow-prereleases: true
 
       - uses: yezz123/setup-uv@v4
+
+      # 1. Test suite with free-threading enabled, but no free-threading-specific plugins
       - name: Run free-threaded tests on ${{ matrix.platform}}
+        if: ${{ matrix.plugin == 'none' }}
         run: uvx nox -s free-threading
+
+      # 2. Test suite with free-threading enabled, plus free-threading-specific plugins
+      - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
+        if: ${{ matrix.plugin != 'none' }}
+        env:
+          UV_INDEX_URL: "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+          UV_PRERELEASE: "allow"
+          UV_INDEX_STRATEGY: "first-index"
+          UV_NO_CACHE: "true"
+        run: |
+          python -VV
+          uv pip install --upgrade --only-binary :all: numpy scipy
+          uv pip install -e .
+          uv pip install pytest ${{ matrix.plugin }}
+          if [ "${{ matrix.plugin }}" == "pytest-run-parallel" ]; then
+            PYTHON_GIL="0" pytest --parallel-threads 5
+          elif [ "${{ matrix.plugin }}" == "pytest-freethreaded" ]; then
+            PYTHON_GIL="0" pytest --threads 5
+          fi

From 3a4d6c69e370b815b58710344168d270cee07234 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sat, 14 Dec 2024 22:11:59 +0530
Subject: [PATCH 14/21] Fix typo in test command

---
 noxfile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/noxfile.py b/noxfile.py
index 39ee9d2e..0be46caa 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -35,7 +35,7 @@ def run_tests(session):
     else:
         session.install("-e", ".[test,scipy]", silent=False)
     session.run(
-        "pytest", "-n", "auto" "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs
+        "pytest", "-n", "auto", "--cov=autograd", "--cov-report=xml", "--cov-append", *session.posargs
     )
 
 

From c7a9f47812cf3c57eb4ea51f01349bfa543b0766 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 03:57:49 +0530
Subject: [PATCH 15/21] Add two more sessions for pytest plugins

---
 noxfile.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 49 insertions(+), 3 deletions(-)

diff --git a/noxfile.py b/noxfile.py
index 0be46caa..8b663915 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -70,6 +70,10 @@ def run_nightly_tests(session):
 # free-threaded Python. This session is similar to the "nightly-tests"
 # session, but it uses a free-threaded Python interpreter. Also, we don't
 # the "test" extra but install the test dependencies manually.
+#
+# When the PYTHON_GIL environment variable is set to 0, we enforce that
+# extension modules that haven't declared themselves as safe to not rely
+# on the GIL are run with the GIL disabled.
 @nox.session(name="free-threading", python=["3.13t"])
 def run_with_free_threaded_python(session):
     """Run tests with free threaded Python (no-GIL)"""
@@ -86,9 +90,51 @@ def run_with_free_threaded_python(session):
         session.install(
             "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
         )
-    # The PYTHON_GIL environment variable is set to 0, which enforces that
-    # extension modules that haven't declared themselves as safe to not rely
-    # on the GIL are run with the GIL disabled.
+    session.run(
+        "pytest",
+        *session.posargs,
+        env={"PYTHON_GIL": "0"},
+    )
+
+
+@nox.session(name="free-threading-pytest-run-parallel", python=["3.13t"])
+def run_pytest_run_in_parallel_plugin(session):
+    """Run stress tests with free threaded Python (no-GIL) using the pytest-run-in-parallel plugin"""
+    session.run("python", "-VV")
+    session.install("-e", ".", silent=False)
+    session.install("pytest", "pytest-run-in-parallel", silent=False)
+
+    # SciPy doesn't have wheels on PyPy
+    if platform.python_implementation() == "PyPy":
+        session.install(
+            "numpy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
+        )
+    else:
+        session.install(
+            "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
+        )
+    session.run(
+        "pytest" * session.posargs,
+        env={"PYTHON_GIL": "0"},
+    )
+
+
+@nox.session(name="free-threading-pytest-freethreaded", python=["3.13t"])
+def run_pytest_freethreaded(session):
+    """Run stress tests with free threaded Python (no-GIL) using the pytest-freethreaded plugin"""
+    session.run("python", "-VV")
+    session.install("-e", ".", silent=False)
+    session.install("pytest", "pytest-freethreaded", silent=False)
+
+    # SciPy doesn't have wheels on PyPy
+    if platform.python_implementation() == "PyPy":
+        session.install(
+            "numpy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
+        )
+    else:
+        session.install(
+            "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
+        )
     session.run(
         "pytest",
         *session.posargs,

From 55ab9622f9b2f2dad2c7d08c92f7d37a2ddb23a1 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 04:00:38 +0530
Subject: [PATCH 16/21] Add GHA matrix for free-threaded plugin execution

---
 .github/workflows/test.yml | 28 +++++++++-------------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 47e71c37..4383f43e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -98,29 +98,19 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
-
-      - uses: yezz123/setup-uv@v4
+      - uses: yezz123/setup-uv@ab6be5a42627f19dc36e57b548592a5e52cece4a # v4.1
 
       # 1. Test suite with free-threading enabled, but no free-threading-specific plugins
       - name: Run free-threaded tests on ${{ matrix.platform}}
         if: ${{ matrix.plugin == 'none' }}
         run: uvx nox -s free-threading
 
-      # 2. Test suite with free-threading enabled, plus free-threading-specific plugins
+      # 2. Test suite with free-threading enabled, plus pytest-run-parallel plugin
+      - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
+        if: ${{ matrix.plugin == 'pytest-run-parallel' }}
+        run: uvx nox -s free-threading-pytest-run-parallel -- --parallel-threads 5
+
+      # 3. Test suite with free-threading enabled, plus pytest-freethreaded plugin
       - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
-        if: ${{ matrix.plugin != 'none' }}
-        env:
-          UV_INDEX_URL: "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-          UV_PRERELEASE: "allow"
-          UV_INDEX_STRATEGY: "first-index"
-          UV_NO_CACHE: "true"
-        run: |
-          python -VV
-          uv pip install --upgrade --only-binary :all: numpy scipy
-          uv pip install -e .
-          uv pip install pytest ${{ matrix.plugin }}
-          if [ "${{ matrix.plugin }}" == "pytest-run-parallel" ]; then
-            PYTHON_GIL="0" pytest --parallel-threads 5
-          elif [ "${{ matrix.plugin }}" == "pytest-freethreaded" ]; then
-            PYTHON_GIL="0" pytest --threads 5
-          fi
+        if: ${{ matrix.plugin == 'pytest-freethreaded' }}
+        run: uvx nox -s free-threading-pytest-freethreaded -- --threads 5

From aff687d478df04cb8ddd310de56a605f952f9645 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 04:14:10 +0530
Subject: [PATCH 17/21] Fix typo in pytest-run-parallel plugin name

---
 noxfile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/noxfile.py b/noxfile.py
index 8b663915..c952e21a 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -102,7 +102,7 @@ def run_pytest_run_in_parallel_plugin(session):
     """Run stress tests with free threaded Python (no-GIL) using the pytest-run-in-parallel plugin"""
     session.run("python", "-VV")
     session.install("-e", ".", silent=False)
-    session.install("pytest", "pytest-run-in-parallel", silent=False)
+    session.install("pytest", "pytest-run-parallel", silent=False)
 
     # SciPy doesn't have wheels on PyPy
     if platform.python_implementation() == "PyPy":

From b79abab33afb23bdcc2be01685b3256f8f021282 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 04:20:03 +0530
Subject: [PATCH 18/21] Fix bad pytest invocation

---
 noxfile.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/noxfile.py b/noxfile.py
index c952e21a..b758ad42 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -114,7 +114,8 @@ def run_pytest_run_in_parallel_plugin(session):
             "numpy", "scipy", "--upgrade", "--only-binary", ":all:", silent=False, env=UV_NIGHTLY_ENV_VARS
         )
     session.run(
-        "pytest" * session.posargs,
+        "pytest",
+        *session.posargs,
         env={"PYTHON_GIL": "0"},
     )
 

From f8735dda0c0563cf9cedc7f1ed8809e634004b79 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 04:20:12 +0530
Subject: [PATCH 19/21] Run both plugins for 100 iterations

---
 .github/workflows/test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4383f43e..5fbe394a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -108,9 +108,10 @@ jobs:
       # 2. Test suite with free-threading enabled, plus pytest-run-parallel plugin
       - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
         if: ${{ matrix.plugin == 'pytest-run-parallel' }}
-        run: uvx nox -s free-threading-pytest-run-parallel -- --parallel-threads 5
+        run: uvx nox -s free-threading-pytest-run-parallel -- --parallel-threads 5 --iterations 100
 
       # 3. Test suite with free-threading enabled, plus pytest-freethreaded plugin
+      # This runs with 200 iterations by default.
       - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
         if: ${{ matrix.plugin == 'pytest-freethreaded' }}
-        run: uvx nox -s free-threading-pytest-freethreaded -- --threads 5
+        run: uvx nox -s free-threading-pytest-freethreaded -- --threads 5 --iterations 100

From c70f7ef58edca9240dd8afb42614dd33ddcf7fd6 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 04:24:53 +0530
Subject: [PATCH 20/21] Remove unneeded `UV_NO_CACHE` env var

---
 noxfile.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/noxfile.py b/noxfile.py
index b758ad42..6203880b 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -7,7 +7,6 @@
     "UV_INDEX_URL": NIGHTLY_INDEX_URL,
     "UV_PRERELEASE": "allow",
     "UV_INDEX_STRATEGY": "first-index",
-    "UV_NO_CACHE": "true",
 }
 
 nox.needs_version = ">=2024.4.15"

From 2cf7b093c748ad1ac9e3764fe569f9dd48afe0cb Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Sun, 15 Dec 2024 04:27:11 +0530
Subject: [PATCH 21/21] Run for 10 and 100 iterations respectively

---
 .github/workflows/test.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5fbe394a..9e203bb9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -100,18 +100,17 @@ jobs:
           allow-prereleases: true
       - uses: yezz123/setup-uv@ab6be5a42627f19dc36e57b548592a5e52cece4a # v4.1
 
-      # 1. Test suite with free-threading enabled, but no free-threading-specific plugins
+      # 1) Test suite with free-threading enabled, but no free-threading-specific plugins
       - name: Run free-threaded tests on ${{ matrix.platform}}
         if: ${{ matrix.plugin == 'none' }}
         run: uvx nox -s free-threading
 
-      # 2. Test suite with free-threading enabled, plus pytest-run-parallel plugin
+      # 2) Test suite with free-threading enabled, plus pytest-run-parallel plugin
       - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
         if: ${{ matrix.plugin == 'pytest-run-parallel' }}
-        run: uvx nox -s free-threading-pytest-run-parallel -- --parallel-threads 5 --iterations 100
+        run: uvx nox -s free-threading-pytest-run-parallel -- -sra --parallel-threads 5 --iterations 10
 
-      # 3. Test suite with free-threading enabled, plus pytest-freethreaded plugin
-      # This runs with 200 iterations by default.
+      # 3) Test suite with free-threading enabled, plus pytest-freethreaded plugin
       - name: Run free-threaded tests with ${{ matrix.plugin }} on ${{ matrix.platform}}
         if: ${{ matrix.plugin == 'pytest-freethreaded' }}
-        run: uvx nox -s free-threading-pytest-freethreaded -- --threads 5 --iterations 100
+        run: uvx nox -s free-threading-pytest-freethreaded -- -sra --threads 5 --iterations 100