Sync visited and shared images across pytest-xdist nodes

.github/workflows/ci_cd.yml

@@ -109,7 +109,7 @@ jobs:
           pip list
 
       - name: Unit Testing
-        run: xvfb-run coverage run --branch --source=pytest_pyvista -m pytest --verbose .
+        run: xvfb-run coverage run --branch --source=pytest_pyvista -m pytest --verbose -n2
       - uses: codecov/codecov-action@v5
         if: matrix.python-version == '3.9'
         name: "Upload coverage to CodeCov"
@@ -188,7 +188,7 @@ jobs:
         working-directory: pyvista
 
       - name: Unit Testing
-        run: xvfb-run python -m pytest -v --allow_useless_fixture --generated_image_dir gen_dir tests/plotting/test_plotting.py
+        run: xvfb-run python -m pytest -v --allow_useless_fixture --generated_image_dir gen_dir tests/plotting/test_plotting.py -n2
         working-directory: pyvista
 
       - name: Upload generated image artifact

pyproject.toml

@@ -40,6 +40,7 @@ tests = [
   "coverage==7.10.1",
   "numpy<2.3",
   "pytest-cov==6.2.1",
+  "pytest-xdist==3.8.0",
   "pytest>=6.2.0",
   "pytest_mock<3.15",
 ]

pytest_pyvista/pytest_pyvista.py

@@ -2,10 +2,12 @@
 
 from __future__ import annotations
 
+import fcntl
 import os
 from pathlib import Path
 import platform
 import shutil
+import tempfile
 from typing import TYPE_CHECKING
 from typing import Callable
 from typing import Literal
@@ -20,8 +22,57 @@
     from collections.abc import Generator
 
 
-VISITED_CACHED_IMAGE_NAMES: set[str] = set()
-SKIPPED_CACHED_IMAGE_NAMES: set[str] = set()
+class _SharedFileSync:
+    """Stores stores a set of strings in a file to multiprocessing sync."""
+
+    def __init__(self, prefix: str) -> None:
+        self._prefix = prefix
+        self._cached_path: Path | None = None
+        self._data: set[str] = set()
+
+    @property
+    def _path(self) -> Path:
+        if self._cached_path is None:
+            shared_dir = Path(tempfile.gettempdir())
+            path = shared_dir / f"{self._prefix}_shared.txt"
+            self._cached_path = path
+            path.touch(exist_ok=True)
+        return self._cached_path
+
+    def add(self, name: str) -> None:
+        self._data.add(name)
+
+    def _read(self) -> set[str]:
+        if os.path.isfile(self._path):  # noqa: PTH113
+            with self._path.open() as f:
+                return {line.strip() for line in f if line.strip()}
+        return set()
+
+    def clear(self) -> None:
+        try:
+            with self._path.open('r+') as f:
+                fcntl.flock(f, fcntl.LOCK_EX)
+                f.truncate(0)
+                fcntl.flock(f, fcntl.LOCK_UN)
+            self._buffer.clear()
+        except OSError as e:  # pragma: no cover
+            raise RuntimeError(f"Failed to clear shared file: {e}")
+
+    def write(self) -> None:
+        with self._path.open("a") as f:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            for name in self._data:
+                f.write(name + "\n")
+            fcntl.flock(f, fcntl.LOCK_UN)
+        self._data.clear()
+
+    @property
+    def data(self) -> set[str]:
+        return self._read()
+
+
+VISITED_CACHED_IMAGE_NAMES = _SharedFileSync("visited_")
+SKIPPED_CACHED_IMAGE_NAMES = _SharedFileSync("skipped_")
 
 
 class RegressionError(RuntimeError):
@@ -358,13 +409,14 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None:  # no
     if config.getoption("disallow_unused_cache"):
         cache_path = Path(_get_option_from_config_or_ini(config, "image_cache_dir"))
         cached_image_names = {f.name for f in cache_path.glob("*.png")}
-        unused_cached_image_names = cached_image_names - VISITED_CACHED_IMAGE_NAMES - SKIPPED_CACHED_IMAGE_NAMES
+
+        unused_cached_image_names = cached_image_names - VISITED_CACHED_IMAGE_NAMES.data - SKIPPED_CACHED_IMAGE_NAMES.data
 
         # Exclude images from skipped tests where multiple images are generated
         unused_skipped = unused_cached_image_names.copy()
         for image_name in unused_cached_image_names:
             base_image_name = _image_name_from_test_name(_test_name_from_image_name(image_name))
-            if base_image_name in SKIPPED_CACHED_IMAGE_NAMES:
+            if base_image_name in SKIPPED_CACHED_IMAGE_NAMES.data:
                 unused_skipped.remove(image_name)
 
         if unused_skipped:
@@ -387,7 +439,9 @@ def _ensure_dir_exists(dirpath: str | Path, msg_name: str) -> None:
     if not Path(dirpath).is_dir():
         msg = f"pyvista test {msg_name}: {dirpath} does not yet exist.  Creating dir."
         warnings.warn(msg, stacklevel=2)
-        Path(dirpath).mkdir(parents=True)
+
+        # exist_ok to allow for multi-threading
+        Path(dirpath).mkdir(exist_ok=True, parents=True)
 
 
 def _get_option_from_config_or_ini(pytestconfig: pytest.Config, option: str, *, is_dir: bool = False):  # noqa: ANN202
@@ -490,3 +544,10 @@ def func_show(*args, **kwargs) -> None:  # noqa: ANN002, ANN003
                 "Fixture `verify_image_cache` is used but no images were generated.\n"
                 "Did you forget to call `show` or `plot`, or set `verify_image_cache.allow_useless_fixture=True`?."
             )
+
+
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:  # noqa: ARG001
+    """Ensure cached image names are written to file."""
+    # This works across multiple pytest-xdist nodes or a single non-parallel session
+    VISITED_CACHED_IMAGE_NAMES.write()
+    SKIPPED_CACHED_IMAGE_NAMES.write()