From ce0f3325e5096cb24ba237b636f7d3424c82aad6 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Mon, 4 Aug 2025 09:33:31 +0200 Subject: [PATCH 1/6] add a sanity check ensuring that Numba threading actually works --- PySDM/backends/numba.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/PySDM/backends/numba.py b/PySDM/backends/numba.py index 930262a280..4f03bae313 100644 --- a/PySDM/backends/numba.py +++ b/PySDM/backends/numba.py @@ -7,6 +7,7 @@ import warnings import numba +import numpy as np from PySDM.backends.impl_numba import methods from PySDM.backends.impl_numba.random import Random as ImportedRandom @@ -59,6 +60,26 @@ def __init__(self, formulae=None, double_precision=True, override_jit_flags=None ) parallel_default = False + if ( + parallel_default + and not numba.config.DISABLE_JIT # pylint: disable=no-member + ): + + @numba.jit(parallel=True, nopython=True) + def fill_array_with_thread_id(arr): + """writes thread id to corresponding array element""" + for i in numba.prange( # pylint: disable=not-an-iterable + numba.get_num_threads() + ): + arr[i] = numba.get_thread_id() + + fill_array_with_thread_id(arr := np.full(numba.get_num_threads(), -1)) + if not max(arr) > 0: + raise ValueError( + "Numba threading enabled but does not work" + " (try other setting of the NUMBA_THREADING_LAYER env var?)" + ) + assert "fastmath" not in (override_jit_flags or {}) self.default_jit_flags = { **JIT_FLAGS, # here parallel=False (for out-of-backend code) From 247cc27c91c0f030344464dc10ebe8e18f5a1d18 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Tue, 5 Aug 2025 10:21:25 +0200 Subject: [PATCH 2/6] skip the new test if user explicitly disables parallelisation --- PySDM/backends/numba.py | 68 +++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/PySDM/backends/numba.py b/PySDM/backends/numba.py index 4f03bae313..ab702db4c3 100644 --- a/PySDM/backends/numba.py +++ b/PySDM/backends/numba.py @@ -44,41 +44,43 @@ def __init__(self, formulae=None, double_precision=True, override_jit_flags=None self.formulae_flattened = self.formulae.flatten parallel_default = True - if platform.machine() == "arm64": - if "CI" not in os.environ: - warnings.warn( - "Disabling Numba threading due to ARM64 CPU (atomics do not work yet)" - ) - parallel_default = False # TODO #1183 - atomics don't work on ARM64! - try: - numba.parfors.parfor.ensure_parallel_support() - except numba.core.errors.UnsupportedParforsError: - if "CI" not in os.environ: - warnings.warn( - "Numba version used does not support parallel for (32 bits?)" - ) - parallel_default = False + if override_jit_flags is not None and 'parallel' in override_jit_flags: + parallel_default = override_jit_flags['parallel'] + + if parallel_default: + if platform.machine() == "arm64": + if "CI" not in os.environ: + warnings.warn( + "Disabling Numba threading due to ARM64 CPU (atomics do not work yet)" + ) + parallel_default = False # TODO #1183 - atomics don't work on ARM64! + + try: + numba.parfors.parfor.ensure_parallel_support() + except numba.core.errors.UnsupportedParforsError: + if "CI" not in os.environ: + warnings.warn( + "Numba version used does not support parallel for (32 bits?)" + ) + parallel_default = False + + if not numba.config.DISABLE_JIT: # pylint: disable=no-member - if ( - parallel_default - and not numba.config.DISABLE_JIT # pylint: disable=no-member - ): - - @numba.jit(parallel=True, nopython=True) - def fill_array_with_thread_id(arr): - """writes thread id to corresponding array element""" - for i in numba.prange( # pylint: disable=not-an-iterable - numba.get_num_threads() - ): - arr[i] = numba.get_thread_id() - - fill_array_with_thread_id(arr := np.full(numba.get_num_threads(), -1)) - if not max(arr) > 0: - raise ValueError( - "Numba threading enabled but does not work" - " (try other setting of the NUMBA_THREADING_LAYER env var?)" - ) + @numba.jit(parallel=True, nopython=True) + def fill_array_with_thread_id(arr): + """writes thread id to corresponding array element""" + for i in numba.prange( # pylint: disable=not-an-iterable + numba.get_num_threads() + ): + arr[i] = numba.get_thread_id() + + fill_array_with_thread_id(arr := np.full(numba.get_num_threads(), -1)) + if not max(arr) > 0: + raise ValueError( + "Numba threading enabled but does not work" + " (try other setting of the NUMBA_THREADING_LAYER env var?)" + ) assert "fastmath" not in (override_jit_flags or {}) self.default_jit_flags = { From f60a7b2a7195783f6a4fee803a9a7a4596526329 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Tue, 5 Aug 2025 10:50:00 +0200 Subject: [PATCH 3/6] precommit --- PySDM/backends/numba.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/PySDM/backends/numba.py b/PySDM/backends/numba.py index ab702db4c3..157fdf6490 100644 --- a/PySDM/backends/numba.py +++ b/PySDM/backends/numba.py @@ -45,9 +45,9 @@ def __init__(self, formulae=None, double_precision=True, override_jit_flags=None parallel_default = True - if override_jit_flags is not None and 'parallel' in override_jit_flags: - parallel_default = override_jit_flags['parallel'] - + if override_jit_flags is not None and "parallel" in override_jit_flags: + parallel_default = override_jit_flags["parallel"] + if parallel_default: if platform.machine() == "arm64": if "CI" not in os.environ: @@ -55,7 +55,7 @@ def __init__(self, formulae=None, double_precision=True, override_jit_flags=None "Disabling Numba threading due to ARM64 CPU (atomics do not work yet)" ) parallel_default = False # TODO #1183 - atomics don't work on ARM64! - + try: numba.parfors.parfor.ensure_parallel_support() except numba.core.errors.UnsupportedParforsError: @@ -64,7 +64,7 @@ def __init__(self, formulae=None, double_precision=True, override_jit_flags=None "Numba version used does not support parallel for (32 bits?)" ) parallel_default = False - + if not numba.config.DISABLE_JIT: # pylint: disable=no-member @numba.jit(parallel=True, nopython=True) @@ -74,7 +74,7 @@ def fill_array_with_thread_id(arr): numba.get_num_threads() ): arr[i] = numba.get_thread_id() - + fill_array_with_thread_id(arr := np.full(numba.get_num_threads(), -1)) if not max(arr) > 0: raise ValueError( From ecfd7dcad0d5912e04085ed4f086676a22d1215a Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Wed, 6 Aug 2025 08:24:03 +0200 Subject: [PATCH 4/6] rename test file --- .../unit_tests/backends/test_ctor_defaults.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 tests/unit_tests/backends/test_ctor_defaults.py diff --git a/tests/unit_tests/backends/test_ctor_defaults.py b/tests/unit_tests/backends/test_ctor_defaults.py deleted file mode 100644 index fcd8da0993..0000000000 --- a/tests/unit_tests/backends/test_ctor_defaults.py +++ /dev/null @@ -1,19 +0,0 @@ -# pylint: disable=missing-module-docstring,missing-class-docstring,missing-function-docstring -import inspect - -from PySDM.backends import CPU, GPU - - -class TestCtorDefaults: - @staticmethod - def test_gpu_ctor_defaults(): - signature = inspect.signature(GPU.__init__) - assert signature.parameters["verbose"].default is False - assert signature.parameters["debug"].default is False - assert signature.parameters["double_precision"].default is False - assert signature.parameters["formulae"].default is None - - @staticmethod - def test_cpu_ctor_defaults(): - signature = inspect.signature(CPU.__init__) - assert signature.parameters["formulae"].default is None From 710b437b03fdba168bab0204cb5cc4c5c28fbde3 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Wed, 6 Aug 2025 08:55:02 +0200 Subject: [PATCH 5/6] add test checking if the threading warning is triggered --- PySDM/backends/numba.py | 5 ++-- .../test_ctor_defaults_and_warnings.py | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/unit_tests/backends/test_ctor_defaults_and_warnings.py diff --git a/PySDM/backends/numba.py b/PySDM/backends/numba.py index 1e40ec14a2..b337862bc6 100644 --- a/PySDM/backends/numba.py +++ b/PySDM/backends/numba.py @@ -7,6 +7,7 @@ import warnings import numba +from numba import prange import numpy as np from PySDM.backends.impl_numba import methods @@ -72,13 +73,13 @@ def __init__( @numba.jit(parallel=True, nopython=True) def fill_array_with_thread_id(arr): """writes thread id to corresponding array element""" - for i in numba.prange( # pylint: disable=not-an-iterable + for i in prange( # pylint: disable=not-an-iterable numba.get_num_threads() ): arr[i] = numba.get_thread_id() fill_array_with_thread_id(arr := np.full(numba.get_num_threads(), -1)) - if not max(arr) > 0: + if not max(arr) == arr[-1] == numba.get_num_threads() - 1: raise ValueError( "Numba threading enabled but does not work" " (try other setting of the NUMBA_THREADING_LAYER env var?)" diff --git a/tests/unit_tests/backends/test_ctor_defaults_and_warnings.py b/tests/unit_tests/backends/test_ctor_defaults_and_warnings.py new file mode 100644 index 0000000000..fe81792ad4 --- /dev/null +++ b/tests/unit_tests/backends/test_ctor_defaults_and_warnings.py @@ -0,0 +1,28 @@ +# pylint: disable=missing-module-docstring,missing-class-docstring,missing-function-docstring +from unittest import mock +import inspect +import pytest + +from PySDM.backends import Numba, ThrustRTC + + +class TestCtorDefaultsAndWarnings: + @staticmethod + def test_gpu_ctor_defaults(): + signature = inspect.signature(ThrustRTC.__init__) + assert signature.parameters["verbose"].default is False + assert signature.parameters["debug"].default is False + assert signature.parameters["double_precision"].default is False + assert signature.parameters["formulae"].default is None + + @staticmethod + def test_cpu_ctor_defaults(): + signature = inspect.signature(Numba.__init__) + assert signature.parameters["formulae"].default is None + + @staticmethod + @mock.patch("PySDM.backends.numba.prange", new=range) + def test_check_numba_threading_warning(): + with pytest.raises(ValueError) as exc_info: + Numba() + assert exc_info.match(r"^Numba threading enabled but does not work") From 8e6de2551a69d33e5677b967c738c10241576d0e Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Wed, 6 Aug 2025 11:05:32 +0200 Subject: [PATCH 6/6] take into account new test file name in pypi.yml --- .github/workflows/pypi.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 367740116d..fc6776141b 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -38,10 +38,10 @@ jobs: unset CI cd ${{ matrix.packages-dir }} python -m build 2>&1 | tee build.log - exit `fgrep -i warning build.log | grep -v impl_numba/warnings.py \ - | grep -v "no previously-included files matching" \ - | grep -v "version of {dist_name} already set" \ - | grep -v -E "UserWarning: version of PySDM(-examples)? already set" \ + exit `fgrep -i warning build.log | fgrep -v warnings.py \ + | fgrep -v "no previously-included files matching" \ + | fgrep -v "version of {dist_name} already set" \ + | fgrep -v -E "UserWarning: version of PySDM(-examples)? already set" \ | wc -l` - run: twine check --strict ${{ matrix.packages-dir }}/dist/* - name: check if version string does not contain PyPI-incompatible + char