diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ee9548b15..2c3973dd8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,6 +53,7 @@ jobs: protonGitRef: - ${{ github.event.inputs.protonBranch || 'main' }} - 0.38.0 + python: [/usr/bin/python3-dbg] env: BuildType: ${{matrix.buildType}} ProtonBuildDir: ${{github.workspace}}/qpid-proton/build @@ -69,6 +70,7 @@ jobs: -DENABLE_FUZZ_TESTING=OFF -DRUNTIME_CHECK=${{matrix.runtimeCheck}} -DBUILD_TLS=ON + -DPython_EXECUTABLE=${{matrix.python}} RouterCMakeExtraArgs: > -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -77,13 +79,12 @@ jobs: -DRUNTIME_CHECK=${{matrix.runtimeCheck}} -DSANITIZE_PYTHON=OFF -DBUILD_BENCHMARKS=ON + -DPython_EXECUTABLE=${{matrix.python}} CCACHE_BASEDIR: ${{github.workspace}} CCACHE_DIR: ${{github.workspace}}/.ccache CCACHE_COMPRESS: 'true' CCACHE_MAXSIZE: '400MB' - SKUPPER_SYSTEM_TEST_SKIP_POLICY: True - SKUPPER_SYSTEM_TEST_SKIP_DELIVERY_ABORT: True steps: # The option to enable + in sed regexps differs by OS so we avoid it, https://github.com/actions/upload-artifact/issues/22 @@ -127,18 +128,12 @@ jobs: - name: Create Build and Install directories run: mkdir -p "${ProtonBuildDir}" "${RouterBuildDir}" "${InstallPrefix}" - - name: Setup python - uses: actions/setup-python@v3 - with: - python-version: 3.6 - architecture: x64 - - - name: Install Python build dependencies - run: python -m pip install setuptools wheel tox - - name: Install Linux build dependencies run: | - sudo apt update; sudo apt install -y libdw-dev swig libpython3-dev libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev ccache ninja-build pixz libbenchmark-dev + sudo apt update; sudo apt install -y libdw-dev swig python3-dbg libpython3-dbg libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev ccache ninja-build pixz libbenchmark-dev + + - name: Install Python build dependencies + run: ${{matrix.python}} -m pip install setuptools wheel tox - name: Zero ccache stats run: ccache -z @@ -207,6 +202,7 @@ jobs: protonGitRef: - ${{ github.event.inputs.protonBranch || 'main' }} - 0.38.0 + python: [/usr/bin/python3-dbg] shard: [1, 2] shards: [2] env: @@ -218,6 +214,11 @@ jobs: LD_LIBRARY_PATH: ${{github.workspace}}/install/lib QPID_SYSTEM_TEST_TIMEOUT: 300 QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True + # the PyMalloc mechanism is incompatible with Valgrind, different mechanism must be set here + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONMALLOC + # https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_python.html#debug-version-of-python-memory-alloc-label + PYTHONMALLOC: malloc_debug + PYTHONTRACEMALLOC: 20 # this enables colored output FORCE_COLOR: 1 COLUMNS: 160 @@ -240,12 +241,6 @@ jobs: with: name: skupper_router_wrk_${{env.JOB_IDENTIFIER}} - - name: Setup python - uses: actions/setup-python@v3 - with: - python-version: 3.6 - architecture: x64 - - name: Install Linux runtime/test dependencies run: | sudo apt update; sudo apt install -y libdw1 libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl ncat gdb elfutils findutils file python3-dbg @@ -253,11 +248,17 @@ jobs: - name: Unpack archive run: tar -I pixz -xf archive.tar.xz + - name: Upgrade pip + run: python3 -m pip install --upgrade pip + - name: Install Python runtime/test dependencies - run: python -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt + run: ${{matrix.python}} -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt - - name: install qpid-proton python wheel - run: python -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl + - name: Replace /usr/bin/python3 with ${{matrix.python}}, for tools such as qdmanage + run: sudo ln -sf ${{matrix.python}} /usr/bin/python3 + + - name: Install qpid-proton python wheel (python3-dbg) + run: ${{matrix.python}} -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl - name: CTest working-directory: ${{env.RouterBuildDir}} @@ -265,7 +266,7 @@ jobs: sudo sysctl -w kernel.core_pattern="coredump.%e.%p" ulimit -c unlimited - ctest --timeout 1200 -V --output-junit=Testing/Test.xml --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j100 ${{env.RouterCTestExtraArgs}} + ctest --timeout 1200 -V --output-junit=Testing/Test.xml --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j6 ${{env.RouterCTestExtraArgs}} - name: Report coredump stacktraces (if any tests failed) if: ${{ failure() }} @@ -310,6 +311,7 @@ jobs: containerTag: ['37'] buildType: [RelWithDebInfo] runtimeCheck: [asan, tsan] + python: [/usr/bin/python3.10d] protonGitRef: - ${{ github.event.inputs.protonBranch || 'main' }} - 0.38.0 @@ -321,6 +323,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: ${{ github.event.inputs.protonBranch || 'main' }} shard: 1 shards: 2 @@ -328,6 +331,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: ${{ github.event.inputs.protonBranch || 'main' }} shard: 2 shards: 2 @@ -335,6 +339,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: 0.38.0 shard: 1 shards: 2 @@ -342,6 +347,7 @@ jobs: container: 'centos' containerTag: stream8 runtimeCheck: OFF + python: /usr/bin/python3.6-debug protonGitRef: 0.38.0 shard: 2 shards: 2 @@ -385,10 +391,11 @@ jobs: -DENABLE_FUZZ_TESTING=OFF -DRUNTIME_CHECK=${{matrix.runtimeCheck}} -DBUILD_TLS=ON + -DPython_EXECUTABLE=${{matrix.python}} RouterCMakeExtraArgs: > -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DPython_EXECUTABLE=/usr/bin/python3 + -DPython_EXECUTABLE=${{matrix.python}} -DRUNTIME_CHECK=${{matrix.runtimeCheck}} CCACHE_BASEDIR: ${{github.workspace}} @@ -397,8 +404,6 @@ jobs: CCACHE_MAXSIZE: '400MB' QPID_SYSTEM_TEST_TIMEOUT: 300 QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True - SKUPPER_SYSTEM_TEST_SKIP_POLICY: True - SKUPPER_SYSTEM_TEST_SKIP_DELIVERY_ABORT: True VERBOSE: 1 steps: @@ -432,10 +437,10 @@ jobs: - name: Install Linux build dependencies run: | - dnf install -y gcc gcc-c++ elfutils-devel cmake libuuid-devel openssl openssl-devel cyrus-sasl-devel cyrus-sasl-plain swig python3-devel python3-pip make libwebsockets-devel libnghttp2-devel ccache libasan libubsan libtsan + dnf install -y gcc gcc-c++ elfutils-devel cmake libuuid-devel openssl openssl-devel cyrus-sasl-devel cyrus-sasl-plain swig python3-devel python3-debug python3-pip make libwebsockets-devel libnghttp2-devel ccache libasan libubsan libtsan - name: Install Python build dependencies - run: python3 -m pip install setuptools wheel tox + run: ${{matrix.python}} -m pip install setuptools wheel tox # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ - name: Prepare ccache timestamp @@ -460,12 +465,10 @@ jobs: - name: Zero ccache stats run: ccache -z - # PROTON-2473: either install `openssl1.1-devel` compat version, or build with `-Wno-error=deprecated-declarations` - name: qpid-proton cmake configure working-directory: ${{env.ProtonBuildDir}} run: > cmake "${{github.workspace}}/qpid-proton" \ - '-DCMAKE_C_FLAGS="-Wno-error=deprecated-declarations"' \ "-DCMAKE_INSTALL_PREFIX=${InstallPrefix}" \ "-DCMAKE_BUILD_TYPE=${BuildType}" \ ${ProtonCMakeExtraArgs} @@ -510,16 +513,24 @@ jobs: - name: Upgrade pip run: python3 -m pip install --upgrade pip + - name: CentOS 8 python3.6-debug needs this to build pip packages from source in next step + if: ${{ matrix.container == 'centos' }} + run: | + dnf install -y libxml2-devel libxslt-devel + - name: Install Python runtime/test dependencies - run: python3 -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt + run: ${{matrix.python}} -m pip install -r ${{github.workspace}}/skupper-router/requirements-dev.txt - name: Install Linux runtime/test dependencies run: | dnf install -y curl nmap-ncat gdb binutils elfutils elfutils-debuginfod-client findutils file dnf debuginfo-install -y python3 + - name: Replace /usr/bin/python3 with ${{matrix.python}}, for tools such as skmanage + run: sudo ln -sf ${{matrix.python}} /usr/bin/python3 + - name: install qpid-proton python wheel - run: python3 -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl + run: ${{matrix.python}} -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl - name: CTest working-directory: ${{env.RouterBuildDir}}