From 3dcc63e22a9a80076ddf25f20ea5b330dd927d06 Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Thu, 14 Aug 2025 16:05:14 -0400 Subject: [PATCH 1/8] Setup a sanitized build for CSP Signed-off-by: Adam Glustein --- CMakeLists.txt | 14 ++++++++++++++ Makefile | 10 ++++++++++ cpp/cmake/modules/Findcsp_autogen.cmake | 14 +++++++++++++- cpp/csp/core/DynamicBitSet.h | 3 ++- cpp/csp/core/Exception.cpp | 2 +- cpp/csp/python/cspbaselibimpl.cpp | 8 ++++++++ cpp/csp/python/npstatsimpl.cpp | 8 ++++---- cpp/tests/engine/test_tick_buffer.cpp | 4 ++++ csp/tests/test_engine.py | 5 +++++ setup.py | 2 ++ 10 files changed, 63 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0e51bca5..e8e3c4f11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,8 @@ option(CSP_MANYLINUX "Build for python's manylinux setup" OFF) option(CSP_USE_VCPKG "Build with vcpkg dependencies" ON) option(CSP_USE_CCACHE "Build with ccache caching" OFF) option(CSP_USE_LD_CLASSIC_MAC "On macOS, link with ld_classic" OFF) +option(CSP_ENABLE_ASAN "Build with address sanitizer" OFF) +option(CSP_ENABLE_UBSAN "Build with undefined behavior sanitizer" OFF) # Extension options option(CSP_BUILD_KAFKA_ADAPTER "Build kafka adapter" ON) @@ -213,6 +215,18 @@ else() endif() endif() +if(CSP_ENABLE_ASAN) + message(STATUS "Enabling Address Sanitizer") + add_compile_options(-fsanitize=address -fno-omit-frame-pointer) + add_link_options(-fsanitize=address -fno-omit-frame-pointer) +endif() + +if(CSP_ENABLE_UBSAN) + message(STATUS "Enabling Undefined Behavior Sanitizer") + add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer) + add_link_options(-fsanitize=undefined -fno-omit-frame-pointer) +endif() + ################################################################################################################################################### # Messages # diff --git a/Makefile b/Makefile index a719bbc22..284278eda 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,18 @@ develop: requirements ## install dependencies and build library build: ## build the library python setup.py build build_ext --inplace +build-sanitizer: ## build the library with ASAN and UBSAN enabled + CSP_ENABLE_ASAN=1 CSP_ENABLE_UBSAN=1 python setup.py build build_ext --inplace + build-debug: ## build the library ( DEBUG ) - May need a make clean when switching from regular build to build-debug and vice versa SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace build-conda: ## build the library in Conda python setup.py build build_ext --csp-no-vcpkg --inplace +build-conda-sanitizer: ## build the library in Conda with ASAN and UBSAN enabled + CSP_ENABLE_ASAN=1 CSP_ENABLE_UBSAN=1 python setup.py build build_ext --csp-no-vcpkg --inplace + install: ## install library python -m pip install . @@ -84,6 +90,10 @@ TEST_ARGS := test-py: ## Clean and Make unit tests python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS) +test-py-sanitizer: ## Clean and Make unit tests with sanitizers enabled + ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true LD_PRELOAD=$$(gcc -print-file-name=libasan.so) \ + python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS) + test-cpp: ## Make C++ unit tests ifneq ($(OS),Windows_NT) for f in ./csp/tests/bin/*; do $$f; done || (echo "TEST FAILED" && exit 1) diff --git a/cpp/cmake/modules/Findcsp_autogen.cmake b/cpp/cmake/modules/Findcsp_autogen.cmake index 3e8284bb9..8e987b1e3 100644 --- a/cpp/cmake/modules/Findcsp_autogen.cmake +++ b/cpp/cmake/modules/Findcsp_autogen.cmake @@ -26,8 +26,20 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU set(CSP_AUTOGEN_PYTHONPATH ${PROJECT_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}:$$PYTHONPATH) endif() + if(CSP_ENABLE_ASAN) + execute_process( + COMMAND gcc -print-file-name=libasan.so + OUTPUT_VARIABLE ASAN_LIB_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + # Turn off leak checks as we are using PyMalloc when we run autogen + set(ASAN_PRELOAD_CMD "ASAN_OPTIONS=detect_leaks=0" "LD_PRELOAD=${ASAN_LIB_PATH}") + else() + set(ASAN_PRELOAD_CMD "") + endif() + add_custom_command(OUTPUT "${CSP_AUTOGEN_CPP_OUT}" "${CSP_AUTOGEN_H_OUT}" - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CSP_AUTOGEN_PYTHONPATH}" ${Python_EXECUTABLE} ${CSP_AUTOGEN_MODULE_PATH} -m ${MODULE_NAME} -d ${CSP_AUTOGEN_DESTINATION_FOLDER} -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS} + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CSP_AUTOGEN_PYTHONPATH}" ${ASAN_PRELOAD_CMD} ${Python_EXECUTABLE} ${CSP_AUTOGEN_MODULE_PATH} -m ${MODULE_NAME} -d ${CSP_AUTOGEN_DESTINATION_FOLDER} -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS} COMMENT "generating csp c++ types from module ${MODULE_NAME}" DEPENDS mkdir_autogen_${MODULE_TARGETNAME} ${CSP_AUTOGEN_MODULE_PATH} diff --git a/cpp/csp/core/DynamicBitSet.h b/cpp/csp/core/DynamicBitSet.h index 5fb120f06..2da4b512b 100644 --- a/cpp/csp/core/DynamicBitSet.h +++ b/cpp/csp/core/DynamicBitSet.h @@ -141,7 +141,8 @@ class DynamicBitSet { node_type * old = m_nodes; m_nodes = new node_type[ newNodes ]; - memcpy( m_nodes, old, m_numNodes * sizeof( node_type ) ); + if( likely( m_numNodes > 0 ) ) + memcpy( m_nodes, old, m_numNodes * sizeof( node_type ) ); memset( m_nodes + m_numNodes, 0, ( newNodes - m_numNodes ) * sizeof( node_type ) ); m_numNodes = newNodes; diff --git a/cpp/csp/core/Exception.cpp b/cpp/csp/core/Exception.cpp index e4480754b..a06c504b5 100644 --- a/cpp/csp/core/Exception.cpp +++ b/cpp/csp/core/Exception.cpp @@ -34,7 +34,7 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) { char *begin_name = 0, *begin_offset = 0; char tmp[1024]; - strncpy( tmp, messages[i], sizeof(tmp) ); + strncpy( tmp, messages[i], sizeof(tmp) - 1 ); tmp[ sizeof( tmp ) - 1 ] = 0; // find parentheses and +address offset surrounding the mangled name: diff --git a/cpp/csp/python/cspbaselibimpl.cpp b/cpp/csp/python/cspbaselibimpl.cpp index 1ada5ced9..0aa7ffc22 100644 --- a/cpp/csp/python/cspbaselibimpl.cpp +++ b/cpp/csp/python/cspbaselibimpl.cpp @@ -273,6 +273,14 @@ DECLARE_CPPNODE( exprtk_impl ) csp.make_passive( inputs ); } + virtual ~exprtk_impl() + { + // Need to release the expression before clearing values/symbol table + // https://github.com/ArashPartow/exprtk/blob/cc1b800c2bd1ac3ac260478c915d2aec6f4eb41c/readme.txt#L909 + s_expr.release(); + s_valuesContainer.clear(); + } + INVOKE() { if( use_trigger ) diff --git a/cpp/csp/python/npstatsimpl.cpp b/cpp/csp/python/npstatsimpl.cpp index f691e8184..8c35e5668 100644 --- a/cpp/csp/python/npstatsimpl.cpp +++ b/cpp/csp/python/npstatsimpl.cpp @@ -1303,12 +1303,12 @@ DECLARE_CPPNODE( _np_arg_min_max ) PyArray_Descr *descr; PyArray_DescrConverter( date_type, &descr ); Py_XDECREF( date_type ); - DateTime * values = new DateTime[s_elem.size()]; + + PyObject * out = PyArray_NewFromDescr( &PyArray_Type, descr, s_shp.m_dims.size(), &s_shp.m_dims[0], NULL, NULL, 0, NULL ); + DateTime * values = static_cast( PyArray_DATA( ( PyArrayObject * )out ) ); for( size_t i = 0; i < s_elem.size(); ++i ) values[i] = s_elem[i].compute_dt(); - - PyObject * out = PyArray_NewFromDescr( &PyArray_Type, descr, s_shp.m_dims.size(), &s_shp.m_dims[0], NULL, values, 0, NULL ); - PyArray_ENABLEFLAGS( ( PyArrayObject * ) out, NPY_ARRAY_OWNDATA ); + RETURN( PyObjectPtr::own( out ) ); } } diff --git a/cpp/tests/engine/test_tick_buffer.cpp b/cpp/tests/engine/test_tick_buffer.cpp index 443639f3f..8d97c0422 100644 --- a/cpp/tests/engine/test_tick_buffer.cpp +++ b/cpp/tests/engine/test_tick_buffer.cpp @@ -98,4 +98,8 @@ TEST( TickBufferTest, test_flatten ) ASSERT_EQ( values_wrap[ i ], i + 3 ); ASSERT_EQ( values_nowrap[ i ], i ); } + + free( values_wrap ); + free( values_nowrap ); + free( values_single ); } diff --git a/csp/tests/test_engine.py b/csp/tests/test_engine.py index 4afe17837..f65c24fa9 100644 --- a/csp/tests/test_engine.py +++ b/csp/tests/test_engine.py @@ -14,6 +14,7 @@ import numpy as np import psutil +import pytest import csp from csp import PushMode, ts @@ -1175,6 +1176,10 @@ def list_comprehension_bug_graph(): rv = csp.run(list_comprehension_bug_graph, starttime=datetime(2020, 1, 1))["Bucket"] self.assertEqual([v[1][0] for v in rv[10:]], list(range(20))) + @unittest.skipIf( + os.environ.get("ASAN_OPTIONS") is not None, + reason="Test skipped when AddressSanitizer is enabled, RSS usage is much larger than usual", + ) def test_alarm_leak(self): """this was a leak in Scheduler.cpp""" diff --git a/setup.py b/setup.py index 11c0df7f6..b33da0819 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,8 @@ ("CSP_BUILD_KAFKA_ADAPTER", "1"), ("CSP_BUILD_PARQUET_ADAPTER", "1"), ("CSP_BUILD_WS_CLIENT_ADAPTER", "1"), + ("CSP_ENABLE_ASAN", "0"), + ("CSP_ENABLE_UBSAN", "0"), # NOTE: # - omit vcpkg, need to test for presence # - omit ccache, need to test for presence From c5bb91e25498ddf7e00991e3b52d53f24f538450 Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Fri, 15 Aug 2025 16:29:17 -0400 Subject: [PATCH 2/8] Add sanitizer support for clang/mac Signed-off-by: Adam Glustein --- Makefile | 14 +++++++++++-- cpp/cmake/modules/Findcsp_autogen.cmake | 26 ++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 284278eda..d93fc2ab2 100644 --- a/Makefile +++ b/Makefile @@ -91,8 +91,18 @@ test-py: ## Clean and Make unit tests python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS) test-py-sanitizer: ## Clean and Make unit tests with sanitizers enabled - ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true LD_PRELOAD=$$(gcc -print-file-name=libasan.so) \ - python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS) + @if [ "$$(uname -s)" = "Darwin" ]; then \ + ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \ + DYLD_INSERT_LIBRARIES=$$($(CXX) -print-file-name=libclang_rt.asan_osx_dynamic.dylib) \ + python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS); \ + elif [ "$$(uname -s)" = "Linux" ]; then \ + ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \ + LD_PRELOAD=$$($(CXX) -print-file-name=libasan.so) \ + python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS); \ + else \ + echo "Unsupported platform: $$(uname -s)"; \ + exit 1; \ + fi test-cpp: ## Make C++ unit tests ifneq ($(OS),Windows_NT) diff --git a/cpp/cmake/modules/Findcsp_autogen.cmake b/cpp/cmake/modules/Findcsp_autogen.cmake index 8e987b1e3..c44a5c163 100644 --- a/cpp/cmake/modules/Findcsp_autogen.cmake +++ b/cpp/cmake/modules/Findcsp_autogen.cmake @@ -27,13 +27,25 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU endif() if(CSP_ENABLE_ASAN) - execute_process( - COMMAND gcc -print-file-name=libasan.so - OUTPUT_VARIABLE ASAN_LIB_PATH - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - # Turn off leak checks as we are using PyMalloc when we run autogen - set(ASAN_PRELOAD_CMD "ASAN_OPTIONS=detect_leaks=0" "LD_PRELOAD=${ASAN_LIB_PATH}") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # Clang - use DYLD_INSERT_LIBRARIES + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libclang_rt.asan_osx_dynamic.dylib + OUTPUT_VARIABLE ASAN_LIB_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(PRELOAD_CMD "DYLD_INSERT_LIBRARIES=${ASAN_LIB_PATH}") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + # GCC - use LD_PRELOAD + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libasan.so + OUTPUT_VARIABLE ASAN_LIB_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(PRELOAD_CMD "LD_PRELOAD=${ASAN_LIB_PATH}") + endif() + # Turn off leak checks as we are using PyMalloc when we run autogen + set(ASAN_PRELOAD_CMD "ASAN_OPTIONS=detect_leaks=0" ${PRELOAD_CMD}) else() set(ASAN_PRELOAD_CMD "") endif() From d5b67557cfbb2aeb62524c40b782e15c49fcb7a0 Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Tue, 26 Aug 2025 18:40:49 -0400 Subject: [PATCH 3/8] Scheduled weekly sanitizer build Signed-off-by: Adam Glustein --- .github/workflows/build.yml | 52 ++++++++++++++++++++++++++++++++----- Makefile | 19 +++++++------- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50895dde8..9528f65f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,8 @@ env: # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule # GitHub Actions schedules can sometimes delay by up to 15 minutes due to platform load FULL_CI_SCHEDULE: '25 8 * * 1,4' + # Run sanitized builds (MacOS/Linux) on Friday at 3:25am EST (08:25 UTC) + SANITIZER_CI_SCHEDULE: '25 8 * * 5' on: push: @@ -33,8 +35,13 @@ on: required: false type: boolean default: false + sanitizer: + description: "Run sanitized build" + required: false + type: boolean + default: false schedule: - - cron: '25 8 * * 1,4' + - cron: '25 8 * * 1,4,5' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -68,9 +75,9 @@ jobs: runs-on: ubuntu-24.04 outputs: - COMMIT_MESSAGE: ${{ steps.setup.outputs.COMMIT_MSG }} + COMMIT_MESSAGE: ${{ steps.getcommitpush.outputs.COMMIT_MSG || steps.getcommitpr.outputs.COMMIT_MSG }} FULL_RUN: ${{ steps.setuppush.outputs.FULL_RUN || steps.setuppr.outputs.FULL_RUN || steps.setupmanual.outputs.FULL_RUN || steps.setupschedule.outputs.FULL_RUN }} - + SANITIZER: ${{ steps.setupmanual.outputs.SANITIZER || steps.setupschedule.outputs.SANITIZER }} steps: - name: Checkout uses: actions/checkout@v4 @@ -79,11 +86,13 @@ jobs: fetch-depth: 2 - name: Get Commit Message - run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD | tr '\n' ' ')" >> $GITHUB_ENV + id: getcommitpush + run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD | tr '\n' ' ')" >> $GITHUB_OUTPUT if: ${{ github.event_name == 'push' }} - name: Get Commit Message - run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD^2 | tr '\n' ' ')" >> $GITHUB_ENV + id: getcommitpr + run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD^2 | tr '\n' ' ')" >> $GITHUB_OUTPUT if: ${{ github.event_name == 'pull_request' }} - name: Display and Setup Build Args (Push) @@ -117,6 +126,7 @@ jobs: echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT env: FULL_RUN: ${{ github.event.inputs.ci-full }} + SANITIZER: ${{ github.event.inputs.sanitizer }} if: ${{ github.event_name == 'workflow_dispatch' }} - name: Display and Setup Build Args (Schedule) @@ -128,6 +138,7 @@ jobs: echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT env: FULL_RUN: ${{ github.event.schedule == env.FULL_CI_SCHEDULE }} + SANITIZER: ${{ github.event.schedule == env.SANITIZER_CI_SCHEDULE }} if: ${{ github.event_name == 'schedule' }} ######################################################## @@ -209,6 +220,8 @@ jobs: - "cp312" is-full-run: - ${{ needs.initialize.outputs.FULL_RUN == 'true' }} + is-sanitizer: + - ${{ needs.initialize.outputs.SANITIZER == 'true' }} exclude: ############################ # Things to always exclude # @@ -261,6 +274,10 @@ jobs: os: windows-2022 python-version: "3.11" + # No sanitizer support for msvc yet (no UBSAN) + - is-sanitizer: true + os: windows-2022 + # avoid unnecessary use of mac resources - is-full-run: false os: macos-13 @@ -302,7 +319,13 @@ jobs: run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-manylinux*" - CIBW_ENVIRONMENT_LINUX: ACLOCAL_PATH="/usr/share/aclocal" CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}" + CIBW_ENVIRONMENT_LINUX: >- + ACLOCAL_PATH="/usr/share/aclocal" + CSP_MANYLINUX="ON" + ${{ needs.initialize.outputs.SANITIZER == 'true' && 'CSP_ENABLE_ASAN="ON" CSP_ENABLE_UBSAN="ON"' || '' }} + CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" + VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" + VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}" CIBW_BUILD_VERBOSITY: 3 if: ${{ runner.os == 'Linux' }} @@ -321,7 +344,11 @@ jobs: run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*" - CIBW_ENVIRONMENT_MACOS: CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" + CIBW_ENVIRONMENT_MACOS: >- + ${{ needs.initialize.outputs.SANITIZER == 'true' && 'CSP_ENABLE_ASAN="ON" CSP_ENABLE_UBSAN="ON"' || '' }} + CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" + VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" + VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" CIBW_ARCHS_MACOS: arm64 CIBW_BUILD_VERBOSITY: 3 if: ${{ matrix.os == 'macos-14' }} @@ -441,6 +468,8 @@ jobs: - 3.12 is-full-run: - ${{ needs.initialize.outputs.FULL_RUN == 'true' }} + is-sanitizer: + - ${{ needs.initialize.outputs.SANITIZER == 'true' }} exclude: ############################################## # Things to exclude if not a full matrix run # @@ -462,6 +491,10 @@ jobs: os: windows-2022 python-version: "3.11" + # No sanitizer support for msvc yet (no UBSAN) + - is-sanitizer: true + os: windows-2022 + # avoid unnecessary use of mac resources - is-full-run: false os: macos-13 @@ -543,6 +576,11 @@ jobs: # Common - name: Python Test Steps run: make test + if: ${{ needs.initialize.outputs.SANITIZER == 'false' }} + + - name: Python Test Steps (Sanitizer) + run: make test-sanitizer + if: ${{ needs.initialize.outputs.SANITIZER == 'true' }} ################################################################ #..............................................................# diff --git a/Makefile b/Makefile index d93fc2ab2..075e86336 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ EXTRA_ARGS := ######### .PHONY: requirements develop build build-debug build-conda install +ASAN := +UBSAN := + requirements: ## install python dev and runtime dependencies ifeq ($(OS),Windows_NT) Powershell.exe -executionpolicy bypass -noprofile .\ci\scripts\windows\make_requirements.ps1 @@ -18,19 +21,13 @@ develop: requirements ## install dependencies and build library python -m pip install -e .[develop] build: ## build the library - python setup.py build build_ext --inplace - -build-sanitizer: ## build the library with ASAN and UBSAN enabled - CSP_ENABLE_ASAN=1 CSP_ENABLE_UBSAN=1 python setup.py build build_ext --inplace + CSP_ENABLE_ASAN=$(ASAN) CSP_ENABLE_UBSAN=$(UBSAN) python setup.py build build_ext --inplace build-debug: ## build the library ( DEBUG ) - May need a make clean when switching from regular build to build-debug and vice versa - SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace + CSP_ENABLE_ASAN=$(ASAN) CSP_ENABLE_UBSAN=$(UBSAN) SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace build-conda: ## build the library in Conda - python setup.py build build_ext --csp-no-vcpkg --inplace - -build-conda-sanitizer: ## build the library in Conda with ASAN and UBSAN enabled - CSP_ENABLE_ASAN=1 CSP_ENABLE_UBSAN=1 python setup.py build build_ext --csp-no-vcpkg --inplace + CSP_ENABLE_ASAN=$(ASAN) CSP_ENABLE_UBSAN=$(UBSAN) python setup.py build build_ext --csp-no-vcpkg --inplace install: ## install library python -m pip install . @@ -84,7 +81,7 @@ checks: check ######### # TESTS # ######### -.PHONY: test-py test-cpp coverage-py test tests +.PHONY: test-py test-cpp test-py-sanitizer coverage-py test test-sanitizer tests TEST_ARGS := test-py: ## Clean and Make unit tests @@ -116,6 +113,8 @@ coverage-py: test: test-cpp test-py ## run the tests +test-sanitizer: test-cpp test-py-sanitizer ## run the tests + # Alias tests: test From 3f71b23f58c7de1c7103d85a2cf4654f610c7f4c Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Wed, 27 Aug 2025 18:33:53 -0400 Subject: [PATCH 4/8] Sanitizer build CI for conda Signed-off-by: Adam Glustein --- .github/workflows/build.yml | 4 ++++ .github/workflows/conda.yml | 46 ++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9528f65f6..af06eb770 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,8 +122,10 @@ jobs: run: | echo "Commit Message: $COMMIT_MSG" echo "Full Run: $FULL_RUN" + echo "Sanitizer: $SANITIZER" echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT + echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT env: FULL_RUN: ${{ github.event.inputs.ci-full }} SANITIZER: ${{ github.event.inputs.sanitizer }} @@ -134,8 +136,10 @@ jobs: run: | echo "Commit Message: $COMMIT_MSG" echo "Full Run: $FULL_RUN" + echo "Sanitizer: $SANITIZER" echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT + echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT env: FULL_RUN: ${{ github.event.schedule == env.FULL_CI_SCHEDULE }} SANITIZER: ${{ github.event.schedule == env.SANITIZER_CI_SCHEDULE }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index c782e1a9d..a0bd390f0 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,5 +1,8 @@ name: Conda End-to-end Test +env: + SANITIZER_CI_SCHEDULE: '25 6 * * 5' + on: push: branches: @@ -18,9 +21,16 @@ on: - README.md - "docs/**" workflow_dispatch: + inputs: + sanitizer: + description: "Run sanitized build" + required: false + type: boolean + default: false schedule: # Run conda CI on Monday and Thursday at 1:25am EST (06:25 UTC) - - cron: '25 6 * * 1,4' + # Run conda sanitized builds on Fridays at 1:25 am EST (06:25 UTC) + - cron: '25 6 * * 1,4,5' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -31,7 +41,32 @@ permissions: checks: write jobs: + initialize: + runs-on: ubuntu-24.04 + outputs: + SANITIZER: ${{ steps.setupmanual.outputs.SANITIZER || steps.setupschedule.outputs.SANITIZER }} + steps: + - name: Display and Setup Build Args (Manual) + id: setupmanual + run: | + echo "Sanitizer: $SANITIZER" + echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT + env: + SANITIZER: ${{ github.event.inputs.sanitizer }} + if: ${{ github.event_name == 'workflow_dispatch' }} + + - name: Display and Setup Build Args (Schedule) + id: setupschedule + run: | + echo "Sanitizer: $SANITIZER" + echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT + env: + SANITIZER: ${{ github.event.schedule == env.SANITIZER_CI_SCHEDULE }} + if: ${{ github.event_name == 'schedule' }} + build: + needs: + - initialize strategy: matrix: os: @@ -80,7 +115,7 @@ jobs: if: ${{ runner.os == 'Windows' }} - name: Python Build Steps - run: make build-conda + run: make build-conda ${{ needs.initialize.outputs.SANITIZER && 'ASAN="ON" UBSAN="ON"' || '' }} shell: micromamba-shell {0} if: ${{ runner.os != 'Windows' }} @@ -94,7 +129,12 @@ jobs: - name: Python Test Steps run: make test shell: micromamba-shell {0} - if: ${{ runner.os != 'Windows' }} + if: ${{ runner.os != 'Windows' && needs.initialize.outputs.SANITIZER == 'false' }} + + - name: Python Test Steps (Sanitizer) + run: make test-sanitizer + shell: micromamba-shell {0} + if: ${{ runner.os != 'Windows' && needs.initialize.outputs.SANITIZER == 'true' }} - name: Python Test Steps ( Windows ) run: make test From 2f827defcae4517b1923fcf1664e8253b7ba5654 Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Fri, 29 Aug 2025 12:23:08 -0400 Subject: [PATCH 5/8] Test fix for sporadic Mac build failure Signed-off-by: Adam Glustein --- Makefile | 1 + cpp/cmake/modules/Findcsp_autogen.cmake | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 527b8a970..c48162b01 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,7 @@ test-py-sanitizer: ## Clean and Make unit tests with sanitizers enabled @if [ "$$(uname -s)" = "Darwin" ]; then \ ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \ DYLD_INSERT_LIBRARIES=$$($(CXX) -print-file-name=libclang_rt.asan_osx_dynamic.dylib) \ + DYLD_FORCE_FLAT_NAMESPACE=1 \ python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS); \ elif [ "$$(uname -s)" = "Linux" ]; then \ ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \ diff --git a/cpp/cmake/modules/Findcsp_autogen.cmake b/cpp/cmake/modules/Findcsp_autogen.cmake index c44a5c163..dd31e2eb7 100644 --- a/cpp/cmake/modules/Findcsp_autogen.cmake +++ b/cpp/cmake/modules/Findcsp_autogen.cmake @@ -34,7 +34,7 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU OUTPUT_VARIABLE ASAN_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE ) - set(PRELOAD_CMD "DYLD_INSERT_LIBRARIES=${ASAN_LIB_PATH}") + set(PRELOAD_CMD "DYLD_INSERT_LIBRARIES=${ASAN_LIB_PATH} DYLD_FORCE_FLAT_NAMESPACE=1") elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") # GCC - use LD_PRELOAD execute_process( From 3f3dabe5bd165ac245d59a37bfa2c7740df7e04b Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Wed, 24 Sep 2025 14:37:04 -0400 Subject: [PATCH 6/8] Remove macOS sanitizer from weekly runs, fails on autogen Signed-off-by: Adam Glustein --- .github/workflows/build.yml | 6 +++++- Makefile | 1 - cpp/cmake/modules/Findcsp_autogen.cmake | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2f28845c..220817768 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -277,10 +277,14 @@ jobs: os: windows-2022 python-version: "3.11" - # No sanitizer support for msvc yet (no UBSAN) + # msvc: no sanitizer support yet - is-sanitizer: true os: windows-2022 + # clang: sanitized build fails on csp_autogen + - is-sanitizer: true + os: macos-14 + # avoid unnecessary use of mac resources - is-full-run: false os: macos-14 diff --git a/Makefile b/Makefile index 9c94bf493..60fdd95c8 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,6 @@ test-py-sanitizer: ## Clean and Make unit tests with sanitizers enabled @if [ "$$(uname -s)" = "Darwin" ]; then \ ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \ DYLD_INSERT_LIBRARIES=$$($(CXX) -print-file-name=libclang_rt.asan_osx_dynamic.dylib) \ - DYLD_FORCE_FLAT_NAMESPACE=1 \ python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS); \ elif [ "$$(uname -s)" = "Linux" ]; then \ ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \ diff --git a/cpp/cmake/modules/Findcsp_autogen.cmake b/cpp/cmake/modules/Findcsp_autogen.cmake index dd31e2eb7..c44a5c163 100644 --- a/cpp/cmake/modules/Findcsp_autogen.cmake +++ b/cpp/cmake/modules/Findcsp_autogen.cmake @@ -34,7 +34,7 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU OUTPUT_VARIABLE ASAN_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE ) - set(PRELOAD_CMD "DYLD_INSERT_LIBRARIES=${ASAN_LIB_PATH} DYLD_FORCE_FLAT_NAMESPACE=1") + set(PRELOAD_CMD "DYLD_INSERT_LIBRARIES=${ASAN_LIB_PATH}") elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") # GCC - use LD_PRELOAD execute_process( From dd12677ee9f18b16a49624f0eca6ee41a3abd182 Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Thu, 25 Sep 2025 08:56:46 -0400 Subject: [PATCH 7/8] Need to install libasan, libubsan specially for RH/Debian builds Signed-off-by: Adam Glustein --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 60fdd95c8..cc2acba8e 100644 --- a/Makefile +++ b/Makefile @@ -213,10 +213,12 @@ dependencies-mac: ## install dependencies for mac brew unlink bison flex && brew link --force bison flex dependencies-debian: ## install dependencies for linux - note that zip is needed by bootstrap_vcpkg.sh, do not remove - apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip zip + apt-get update + apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip zip gcc libasan8 libubsan1 dependencies-fedora: ## install dependencies for linux - note that zip is needed by bootstrap_vcpkg.sh, do not remove - yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip zip + yum check-update + yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip zip gcc libasan libubsan dependencies-vcpkg: ## install dependencies via vcpkg cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install From 83fb16ef445289e5c2bb9972e46c5fd3c9263389 Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Thu, 25 Sep 2025 17:11:48 -0400 Subject: [PATCH 8/8] Remove weekly sanitized vcpkg build, only use conda Signed-off-by: Adam Glustein --- .github/workflows/build.yml | 51 +++---------------------------------- Makefile | 6 ++--- 2 files changed, 5 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aed062031..fbf49aba6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,8 +6,6 @@ env: # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule # GitHub Actions schedules can sometimes delay by up to 15 minutes due to platform load FULL_CI_SCHEDULE: '25 8 * * 1,4' - # Run sanitized builds (MacOS/Linux) on Friday at 3:25am EST (08:25 UTC) - SANITIZER_CI_SCHEDULE: '25 8 * * 5' on: push: @@ -35,13 +33,8 @@ on: required: false type: boolean default: false - sanitizer: - description: "Run sanitized build" - required: false - type: boolean - default: false schedule: - - cron: '25 8 * * 1,4,5' + - cron: '25 8 * * 1,4' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -77,7 +70,6 @@ jobs: outputs: COMMIT_MESSAGE: ${{ steps.getcommitpush.outputs.COMMIT_MSG || steps.getcommitpr.outputs.COMMIT_MSG }} FULL_RUN: ${{ steps.setuppush.outputs.FULL_RUN || steps.setuppr.outputs.FULL_RUN || steps.setupmanual.outputs.FULL_RUN || steps.setupschedule.outputs.FULL_RUN }} - SANITIZER: ${{ steps.setupmanual.outputs.SANITIZER || steps.setupschedule.outputs.SANITIZER }} steps: - name: Checkout uses: actions/checkout@v5 @@ -122,13 +114,10 @@ jobs: run: | echo "Commit Message: $COMMIT_MSG" echo "Full Run: $FULL_RUN" - echo "Sanitizer: $SANITIZER" echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT - echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT env: FULL_RUN: ${{ github.event.inputs.ci-full }} - SANITIZER: ${{ github.event.inputs.sanitizer }} if: ${{ github.event_name == 'workflow_dispatch' }} - name: Display and Setup Build Args (Schedule) @@ -136,13 +125,10 @@ jobs: run: | echo "Commit Message: $COMMIT_MSG" echo "Full Run: $FULL_RUN" - echo "Sanitizer: $SANITIZER" echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT - echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT env: FULL_RUN: ${{ github.event.schedule == env.FULL_CI_SCHEDULE }} - SANITIZER: ${{ github.event.schedule == env.SANITIZER_CI_SCHEDULE }} if: ${{ github.event_name == 'schedule' }} ######################################################## @@ -223,8 +209,6 @@ jobs: - "cp312" is-full-run: - ${{ needs.initialize.outputs.FULL_RUN == 'true' }} - is-sanitizer: - - ${{ needs.initialize.outputs.SANITIZER == 'true' }} exclude: ############################ # Things to always exclude # @@ -277,14 +261,6 @@ jobs: os: windows-2022 python-version: "3.11" - # msvc: no sanitizer support yet - - is-sanitizer: true - os: windows-2022 - - # clang: sanitized build fails on csp_autogen - - is-sanitizer: true - os: macos-14 - # avoid unnecessary use of mac resources - is-full-run: false os: macos-14 @@ -323,13 +299,7 @@ jobs: run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-manylinux*" - CIBW_ENVIRONMENT_LINUX: >- - ACLOCAL_PATH="/usr/share/aclocal" - CSP_MANYLINUX="ON" - ${{ needs.initialize.outputs.SANITIZER == 'true' && 'CSP_ENABLE_ASAN="ON" CSP_ENABLE_UBSAN="ON"' || '' }} - CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" - VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" - VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}" + CIBW_ENVIRONMENT_LINUX: ACLOCAL_PATH="/usr/share/aclocal" CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}" CIBW_BUILD_VERBOSITY: 3 if: ${{ runner.os == 'Linux' }} @@ -339,11 +309,7 @@ jobs: run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*" - CIBW_ENVIRONMENT_MACOS: >- - ${{ needs.initialize.outputs.SANITIZER == 'true' && 'CSP_ENABLE_ASAN="ON" CSP_ENABLE_UBSAN="ON"' || '' }} - CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" - VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" - VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" + CIBW_ENVIRONMENT_MACOS: CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" CIBW_ARCHS_MACOS: arm64 CIBW_BUILD_VERBOSITY: 3 if: ${{ matrix.os == 'macos-14' }} @@ -462,8 +428,6 @@ jobs: - 3.12 is-full-run: - ${{ needs.initialize.outputs.FULL_RUN == 'true' }} - is-sanitizer: - - ${{ needs.initialize.outputs.SANITIZER == 'true' }} exclude: ############################################## # Things to exclude if not a full matrix run # @@ -485,10 +449,6 @@ jobs: os: windows-2022 python-version: "3.11" - # No sanitizer support for msvc yet (no UBSAN) - - is-sanitizer: true - os: windows-2022 - # avoid unnecessary use of mac resources - is-full-run: false os: macos-14 @@ -567,11 +527,6 @@ jobs: # Common - name: Python Test Steps run: make test - if: ${{ needs.initialize.outputs.SANITIZER == 'false' }} - - - name: Python Test Steps (Sanitizer) - run: make test-sanitizer - if: ${{ needs.initialize.outputs.SANITIZER == 'true' }} ################################################################ #..............................................................# diff --git a/Makefile b/Makefile index cc2acba8e..60fdd95c8 100644 --- a/Makefile +++ b/Makefile @@ -213,12 +213,10 @@ dependencies-mac: ## install dependencies for mac brew unlink bison flex && brew link --force bison flex dependencies-debian: ## install dependencies for linux - note that zip is needed by bootstrap_vcpkg.sh, do not remove - apt-get update - apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip zip gcc libasan8 libubsan1 + apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip zip dependencies-fedora: ## install dependencies for linux - note that zip is needed by bootstrap_vcpkg.sh, do not remove - yum check-update - yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip zip gcc libasan libubsan + yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip zip dependencies-vcpkg: ## install dependencies via vcpkg cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install