From 0271d3d894df4fd67357fc375db30be04185d574 Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Wed, 13 Apr 2022 07:18:08 -0700 Subject: [PATCH] test: ctest is for unit tests and integration tests must be run standalone (#3866) * test: ctest is for unit tests and integration tests must be run standalone * fixes * build fixes * ignore example 64 on macos * use a test list to target just our test list * fix dir * dont double run test --- .github/workflows/build_macos.yml | 8 +++++- .github/workflows/build_windows_cmake.yml | 3 +- .github/workflows/vendor_build.yml | 13 ++++++--- .scripts/linux/build-minimal.sh | 4 +-- .scripts/linux/build-with-coverage.sh | 2 +- .scripts/linux/test-privacy_activation.sh | 2 +- .scripts/linux/test-with-coverage.sh | 6 ++-- .scripts/linux/test.sh | 12 +++++--- .scripts/linux/unit-tests-valgrind.sh | 3 +- CMakeLists.txt | 6 ---- Makefile | 2 +- cmake/VowpalWabbitUtils.cmake | 12 +++++++- test/CMakeLists.txt | 35 ----------------------- test/unit_test/CMakeLists.txt | 1 + 14 files changed, 48 insertions(+), 61 deletions(-) diff --git a/.github/workflows/build_macos.yml b/.github/workflows/build_macos.yml index bbea5fb2a52..1bcaa14c318 100644 --- a/.github/workflows/build_macos.yml +++ b/.github/workflows/build_macos.yml @@ -25,4 +25,10 @@ jobs: - name: Build run: cmake --build build --target all - name: Test - run: ctest --test-dir build --extra-verbose --output-on-failure + run: | + # Run unit tests + ctest --test-dir build --extra-verbose --output-on-failure --label-regex VWTestList + # Run integration tests + python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty + python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread + python3 test/run_tests.py --ignore_dirty --test_spec test/slow.vwtest.json --timeout 180 diff --git a/.github/workflows/build_windows_cmake.yml b/.github/workflows/build_windows_cmake.yml index 6d22a25f2ec..bcd693bfdf8 100644 --- a/.github/workflows/build_windows_cmake.yml +++ b/.github/workflows/build_windows_cmake.yml @@ -58,4 +58,5 @@ jobs: cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }} - name: Run tests run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }} --target test_with_output + cd ${{ env.CMAKE_BUILD_DIR }} + ctest --verbose --output-on-failure -C ${{ matrix.build_config }} --label-regex VWTestList diff --git a/.github/workflows/vendor_build.yml b/.github/workflows/vendor_build.yml index 2bf908911c3..21af1f5dacd 100644 --- a/.github/workflows/vendor_build.yml +++ b/.github/workflows/vendor_build.yml @@ -45,7 +45,8 @@ jobs: - name: Test run: | python3 test/run_tests.py -f --clean_dirty -E 0.001 - ./build/test/unit_test/vw-unit-test.out + cd build + ctest --verbose --output-on-failure --label-regex VWTestList build_vendor_windows: name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.msvc.standalone runs-on: ${{matrix.os}} @@ -104,7 +105,9 @@ jobs: # TODO: Look into test 67 failure run: python3 ${{ env.SOURCE_DIR }}/test/run_tests.py -f --clean_dirty -E 0.01 --skip_spanning_tree_tests --vw_bin_path ${{ env.CMAKE_BUILD_DIR }}/vowpalwabbit/${{ matrix.build_type }}/vw.exe --skip_test 67 - name: Test unit tests - run: ${{ env.CMAKE_BUILD_DIR }}/test/unit_test/${{ matrix.build_type }}/vw-unit-test.out.exe + run: | + cd ${{ env.CMAKE_BUILD_DIR }} + ctest --verbose --output-on-failure --config ${{ matrix.build_type }} --label-regex VWTestList build_vendor_macos: runs-on: ${{matrix.os}} name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.AppleClang.standalone @@ -130,8 +133,10 @@ jobs: -DVW_BOOST_MATH_SYS_DEP=Off -DVW_INSTALL=Off - name: Build - run: cmake --build build + run: | + cmake --build build - name: Test run: | python3 test/run_tests.py -f --clean_dirty -E 0.001 - ./build/test/unit_test/vw-unit-test.out + cd build + ctest --verbose --output-on-failure --label-regex VWTestList diff --git a/.scripts/linux/build-minimal.sh b/.scripts/linux/build-minimal.sh index e3680f941f5..eb226565c58 100755 --- a/.scripts/linux/build-minimal.sh +++ b/.scripts/linux/build-minimal.sh @@ -9,5 +9,5 @@ cd $REPO_DIR # If parameter 1 is not supplied, it defaults to Release BUILD_CONFIGURATION=${1:-Release} -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DBUILD_FLATBUFFERS=On -cmake --build build --target vw-bin vw-unit-test.out vw_c_api_unit_test spanning_tree to_flatbuff +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DBUILD_FLATBUFFERS=On -DVW_UNIT_TEST_WITH_VALGRIND_INTERNAL=On +cmake --build build diff --git a/.scripts/linux/build-with-coverage.sh b/.scripts/linux/build-with-coverage.sh index 04b70eb2a89..549c56bd758 100755 --- a/.scripts/linux/build-with-coverage.sh +++ b/.scripts/linux/build-with-coverage.sh @@ -7,4 +7,4 @@ REPO_DIR=$SCRIPT_DIR/../../ cd $REPO_DIR cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DWARNINGS=OFF -DBUILD_JAVA=Off -DBUILD_PYTHON=Off -DBUILD_TESTING=On -DBUILD_FLATBUFFERS=On -cmake --build build --target vw-bin spanning_tree vw-unit-test.out +cmake --build build --target vw-bin spanning_tree vw-unit-test.out vw_slim_test diff --git a/.scripts/linux/test-privacy_activation.sh b/.scripts/linux/test-privacy_activation.sh index de7260c1dd7..cfe0cccec83 100755 --- a/.scripts/linux/test-privacy_activation.sh +++ b/.scripts/linux/test-privacy_activation.sh @@ -7,7 +7,7 @@ REPO_DIR=$SCRIPT_DIR/../../ cd $REPO_DIR cd test -python3 run_tests.py -f --skip_spanning_tree_tests -j $(nproc) --test_spec privacy_activation.vwtest.json +python3 run_tests.py -f --skip_spanning_tree_tests --test_spec privacy_activation.vwtest.json cd ../build ./test/unit_test/vw-unit-test.out --run_test=test_feature_is_activated*,test_feature_not_activated*,test_feature_could_be_activated_but_feature_not_initialized* \ No newline at end of file diff --git a/.scripts/linux/test-with-coverage.sh b/.scripts/linux/test-with-coverage.sh index 9ba9db0ca00..96acc5a33ec 100755 --- a/.scripts/linux/test-with-coverage.sh +++ b/.scripts/linux/test-with-coverage.sh @@ -6,6 +6,8 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_DIR=$SCRIPT_DIR/../../ cd $REPO_DIR -cd test +cd $REPO_DIR/build +ctest --verbose --output-on-failure --label-regex VWTestList + +cd $REPO_DIR/test python3 run_tests.py -f -j $(nproc) --include_flatbuffers --clean_dirty -E 0.001 -../build/test/unit_test/vw-unit-test.out diff --git a/.scripts/linux/test.sh b/.scripts/linux/test.sh index a926ce9eb52..17d4d0d6a48 100755 --- a/.scripts/linux/test.sh +++ b/.scripts/linux/test.sh @@ -6,8 +6,12 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_DIR=$SCRIPT_DIR/../../ cd $REPO_DIR -cd test -python3 run_tests.py -f --epsilon 1e-3 --include_flatbuffers +# Run unit tests +cd $REPO_DIR/build +ctest --verbose --output-on-failure --label-regex VWTestList -cd ../build -ctest --verbose --output-on-failure +# Run integration tests +cd $REPO_DIR +python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --include_flatbuffers +python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread --include_flatbuffers +python3 test/run_tests.py --ignore_dirty --test_spec test/slow.vwtest.json --timeout 180 diff --git a/.scripts/linux/unit-tests-valgrind.sh b/.scripts/linux/unit-tests-valgrind.sh index 56d8addedb8..6e2ce7b7a45 100755 --- a/.scripts/linux/unit-tests-valgrind.sh +++ b/.scripts/linux/unit-tests-valgrind.sh @@ -7,5 +7,4 @@ REPO_DIR=$SCRIPT_DIR/../../ cd $REPO_DIR cd build -valgrind --quiet --error-exitcode=100 --track-origins=yes --leak-check=full ./test/unit_test/vw-unit-test.out -valgrind --quiet --error-exitcode=100 --track-origins=yes --leak-check=full ./bindings/c/test/vw_c_api_unit_test +ctest --verbose --output-on-failure --label-regex VWTestList diff --git a/CMakeLists.txt b/CMakeLists.txt index 25c5238466d..d27128646a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -330,12 +330,6 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING) set(cmake_test_verbosity "") endif() - # This target ensures all dependencies are built and also uses verbose mode allowing the test output to be seen. - add_custom_target(test_with_output - COMMAND ${CMAKE_CTEST_COMMAND} ${cmake_test_configuration} ${cmake_test_verbosity} --output-on-failure - DEPENDS spanning_tree vw-unit-test.out vw-bin ${vw_c_api_unit_test_target} ${vw_cs_unittest_targets} - ) - # Don't offer these make dependent targets on Windows if(NOT WIN32) # make bigtests BIG_TEST_ARGS="" diff --git a/Makefile b/Makefile index b3f2f6b7c1f..a4ed6933e48 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ java_build: test_build: @echo "vw running test-suite..." - cd build; make -j$(shell cat ./build/nprocs.txt) test_with_output + cd build; make -j$(shell cat ./build/nprocs.txt) all; make test unit_test_build: cd build/test/unit_test; make -j$(shell cat ./build/nprocs.txt) vw-unit-test.out test diff --git a/cmake/VowpalWabbitUtils.cmake b/cmake/VowpalWabbitUtils.cmake index beb02215a6c..1ed512c8793 100644 --- a/cmake/VowpalWabbitUtils.cmake +++ b/cmake/VowpalWabbitUtils.cmake @@ -1,6 +1,11 @@ include(CMakeParseArguments) include(GNUInstallDirs) +option(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL "Internal flag." OFF) +if(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL) + find_program(VALGRIND "valgrind" REQUIRED) +endif() + # Given a lib name writes to OUTPUT what the correspinding target name will be function(vw_get_lib_target OUTPUT LIB_NAME) set(${OUTPUT} vw_${LIB_NAME} PARENT_SCOPE) @@ -224,6 +229,11 @@ function(vw_add_test_executable) gmock ) target_compile_definitions(${FULL_TEST_NAME} PRIVATE ${VW_TEST_COMPILE_DEFS}) - add_test(NAME ${FULL_TEST_NAME} COMMAND ${FULL_TEST_NAME}) + if(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL) + add_test(NAME ${FULL_TEST_NAME} COMMAND ${VALGRIND} $) + else() + add_test(NAME ${FULL_TEST_NAME} COMMAND ${FULL_TEST_NAME}) + endif() + set_tests_properties(${FULL_TEST_NAME} PROPERTIES LABELS "VWTestList") endif() endfunction() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c4538339a22..eeea053dc44 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,37 +1,2 @@ add_subdirectory(unit_test) add_subdirectory(tools) - -set(FB_FLAG "") # flatbuffers are off by default -if(BUILD_FLATBUFFERS) - set(FB_FLAG "--include_flatbuffers") # include flatbuffer tests in RunTests -endif() - -# Don't offer target on Windows -if(NOT WIN32) - add_test( - NAME RunTests_pass_1_onethread - COMMAND python3 run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread ${FB_FLAG} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) - - add_test( - NAME RunTests_pass_2 - COMMAND python3 run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty ${FB_FLAG} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) - - add_test( - NAME slow_tests - COMMAND python3 run_tests.py --ignore_dirty --test_spec ${CMAKE_CURRENT_SOURCE_DIR}/slow.vwtest.json --timeout 180 - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) - - if(BUILD_PRIVACY_ACTIVATION) - add_test( - NAME privacy_activation_tests - - COMMAND python3 run_tests.py --ignore_dirty --test_spec ${CMAKE_CURRENT_SOURCE_DIR}/privacy_activation.vwtest.json - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) - endif() -endif() diff --git a/test/unit_test/CMakeLists.txt b/test/unit_test/CMakeLists.txt index 8fc589b77ea..8e7e01f0d24 100644 --- a/test/unit_test/CMakeLists.txt +++ b/test/unit_test/CMakeLists.txt @@ -96,3 +96,4 @@ add_test( COMMAND $ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) +set_tests_properties(vw_unit_test PROPERTIES LABELS "VWTestList") \ No newline at end of file