From 4cd8c0d08c6428fd1b2d246e003e6d7791aead32 Mon Sep 17 00:00:00 2001
From: Russell Keith-Magee <russell@keith-magee.com>
Date: Thu, 27 Mar 2025 08:39:48 +0800
Subject: [PATCH 1/2] Correct Github Actions CI instability for iOS.

---
 .github/workflows/test.yml |  2 +-
 bin/run_tests.py           | 34 +++++++++++++++++++++++++++++++---
 pyproject.toml             |  4 +++-
 test/test_ios.py           | 13 ++++++++-----
 4 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6d690562d..23c054423 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,7 +36,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14]
+        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-15]
         python_version: ['3.13']
         include:
         - os: ubuntu-latest
diff --git a/bin/run_tests.py b/bin/run_tests.py
index 2aed4f01a..bbfbc25a5 100755
--- a/bin/run_tests.py
+++ b/bin/run_tests.py
@@ -34,15 +34,39 @@
         if args.run_podman:
             unit_test_args += ["--run-podman"]
 
+    print(
+        "\n\n================================== UNIT TESTS ==================================",
+        flush=True,
+    )
     subprocess.run(unit_test_args, check=True)
 
-    # integration tests
+    # Run the serial integration tests without multiple processes
+    serial_integration_test_args = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-m",
+        "serial",
+        "-x",
+        "--durations",
+        "0",
+        "--timeout=2400",
+        "test",
+        "-vv",
+    ]
+    print(
+        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
+        flush=True,
+    )
+    subprocess.run(serial_integration_test_args, check=True)
+
+    # Non-serial integration tests
     integration_test_args = [
         sys.executable,
         "-m",
         "pytest",
-        "--dist",
-        "loadgroup",
+        "-m",
+        "not serial",
         f"--numprocesses={args.num_processes}",
         "-x",
         "--durations",
@@ -55,4 +79,8 @@
     if sys.platform.startswith("linux") and args.run_podman:
         integration_test_args += ["--run-podman"]
 
+    print(
+        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
+        flush=True,
+    )
     subprocess.run(integration_test_args, check=True)
diff --git a/pyproject.toml b/pyproject.toml
index 0319e2e09..e3efab7ca 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -102,7 +102,9 @@ junit_family = "xunit2"
 xfail_strict = true
 filterwarnings = ["error"]
 log_cli_level = "info"
-
+markers = [
+    "serial: tests that must *not* be run in parallel (deselect with '-m \"not serial\"')",
+]
 
 [tool.mypy]
 python_version = "3.11"
diff --git a/test/test_ios.py b/test/test_ios.py
index 6dc7b40d3..ca6bc7ab2 100644
--- a/test/test_ios.py
+++ b/test/test_ios.py
@@ -20,10 +20,12 @@ def test_platform(self):
 """
 
 
-# iOS tests shouldn't be run in parallel, because they're dependent on starting
-# a simulator. It's *possible* to start multiple simulators, but not advisable
-# to start as many simulators as there are CPUs on the test machine.
-@pytest.mark.xdist_group(name="ios")
+# iOS tests shouldn't be run in parallel, because they're dependent on calling
+# Xcode, and starting a simulator. These are both multi-threaded operations, and
+# it's easy to overload the CI machine if there are multiple test processes
+# running multithreaded processes. Therefore, they're put in the serial group,
+# which is guaranteed to run single-process.
+@pytest.mark.serial
 @pytest.mark.parametrize(
     "build_config",
     [
@@ -48,6 +50,7 @@ def test_ios_platforms(tmp_path, build_config):
             "CIBW_BUILD": "cp313-*",
             "CIBW_TEST_SOURCES": "tests",
             "CIBW_TEST_COMMAND": "unittest discover tests test_platform.py",
+            "CIBW_BUILD_VERBOSITY": "1",
             **build_config,
         },
     )
@@ -71,7 +74,7 @@ def test_ios_platforms(tmp_path, build_config):
     assert set(actual_wheels) == expected_wheels
 
 
-@pytest.mark.xdist_group(name="ios")
+@pytest.mark.serial
 def test_no_test_sources(tmp_path, capfd):
     if utils.platform != "macos":
         pytest.skip("this test can only run on macOS")

From e5d692d5f8904a09a74d8900c3cf0f8750576885 Mon Sep 17 00:00:00 2001
From: Russell Keith-Magee <russell@keith-magee.com>
Date: Thu, 27 Mar 2025 10:08:44 +0800
Subject: [PATCH 2/2] Add a dummy serial test so CircleCI passes serial test
 discovery.

---
 test/test_0_basic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_0_basic.py b/test/test_0_basic.py
index 4ef40a2c2..6a7732031 100644
--- a/test/test_0_basic.py
+++ b/test/test_0_basic.py
@@ -18,6 +18,15 @@
 )
 
 
+@pytest.mark.serial
+def test_dummy_serial():
+    """A no-op test to ensure that at least one serial test is always found.
+
+    Without this no-op test, CI fails on CircleCI because no serial tests are
+    found, and pytest errors if a test suite finds no tests.
+    """
+
+
 def test(tmp_path, build_frontend_env, capfd):
     project_dir = tmp_path / "project"
     basic_project.generate(project_dir)