Try combining test jobs in CI

ci/default.yml

@@ -8,7 +8,7 @@ include:
   rules:
     - if: $BACKEND == 'dace_gpu' || $BACKEND == 'gtfn_gpu'
       variables:
-        NUM_PROCESSES: 8
+        NUM_PROCESSES: 14
         SLURM_TIMELIMIT: '00:30:00'
     - if: $COMPONENT == 'dycore'
       variables:
@@ -18,7 +18,7 @@ include:
         SLURM_TIMELIMIT: '00:05:00'
   parallel:
     matrix:
-      - COMPONENT: [diffusion, dycore, microphysics, muphys, common, driver]
+      - COMPONENT: [all]
         BACKEND: [gtfn_gpu]
         GRID: [simple, icon_regional]
 # test_model_stencils_x86_64:
@@ -47,8 +47,8 @@ test_tools_datatests_aarch64:
       when: never  # run only in daily CI, to save compute resources
     - if: $BACKEND == 'dace_gpu' || $BACKEND == 'gtfn_gpu'
       variables:
-        NUM_PROCESSES: 8
-        SLURM_TIMELIMIT: '00:30:00'
+        NUM_PROCESSES: 14
+        SLURM_TIMELIMIT: '01:30:00'
     - if: $COMPONENT == 'driver' || $COMPONENT == 'dycore' || $COMPONENT == 'muphys'
       variables:
         SLURM_TIMELIMIT: '00:30:00'
@@ -60,8 +60,9 @@ test_tools_datatests_aarch64:
         SLURM_TIMELIMIT: '00:10:00'
   parallel:
     matrix:
-      - COMPONENT: [advection, diffusion, dycore, microphysics, muphys, common, driver, standalone_driver]
-        BACKEND: [embedded, dace_gpu, gtfn_cpu, gtfn_gpu]
+      - COMPONENT: [all]
+        # BACKEND: [embedded, dace_gpu, gtfn_cpu, gtfn_gpu]
+        BACKEND: [dace_gpu, gtfn_gpu]
         LEVEL: [integration]
 # test_model_datatests_x86_64:
 #   extends: [.test_model_datatests, .test_template_x86_64]

noxfile.py

@@ -24,6 +24,7 @@
 
 # -- Parameter sets --
 ModelSubpackagePath: TypeAlias = Literal[
+    "all",
     "atmosphere/advection",
     "atmosphere/diffusion",
     "atmosphere/dycore",
@@ -152,9 +153,10 @@ def test_model(
     _install_session_venv(session, extras=["fortran", "io", "testing"], groups=["test"])
 
     pytest_args = _selection_to_pytest_args(selection)
-    with session.chdir(f"model/{subpackage}"):
+    test_dir = "model" if subpackage == "all" else f"model/{subpackage}"
+    with session.chdir(test_dir):
         session.run(
-            *f"pytest -sv --benchmark-disable -n {os.environ.get('NUM_PROCESSES', 'auto')}".split(),
+            *f"pytest -sv --benchmark-disable --dist=worksteal -n {os.environ.get('NUM_PROCESSES', 'auto')}".split(),
             *pytest_args,
             *session.posargs,
             success_codes=[0, NO_TESTS_COLLECTED_EXIT_CODE],
@@ -188,7 +190,7 @@ def test_tools(session: nox.Session, datatest: bool) -> None:
 
     with session.chdir("tools"):
         session.run(
-            *f"pytest -sv --benchmark-disable -n {os.environ.get('NUM_PROCESSES', 'auto')} {'--datatest-only' if datatest else '--datatest-skip'}".split(),
+            *f"pytest -sv --benchmark-disable --dist=worksteal -n {os.environ.get('NUM_PROCESSES', 'auto')} {'--datatest-only' if datatest else '--datatest-skip'}".split(),
             *session.posargs,
         )