test: ctest is for unit tests and integration tests must be run stand…

…alone (VowpalWabbit#3866)

* test: ctest is for unit tests and integration tests must be run standalone

* fixes

* build fixes

* ignore example 64 on macos

* use a test list to target just our test list

* fix dir

* dont double run test

.github/workflows/build_macos.yml

@@ -25,4 +25,10 @@ jobs:
       - name: Build
         run: cmake --build build --target all
       - name: Test
-        run: ctest --test-dir build --extra-verbose --output-on-failure
+        run: |
+          # Run unit tests
+          ctest --test-dir build --extra-verbose --output-on-failure --label-regex VWTestList
+          # Run integration tests
+          python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty
+          python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread
+          python3 test/run_tests.py --ignore_dirty --test_spec test/slow.vwtest.json --timeout 180

.github/workflows/build_windows_cmake.yml

@@ -58,4 +58,5 @@ jobs:
           cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }}
       - name: Run tests
         run: |
-          cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }} --target test_with_output
+          cd ${{ env.CMAKE_BUILD_DIR }}
+          ctest --verbose --output-on-failure -C ${{ matrix.build_config }} --label-regex VWTestList

.github/workflows/vendor_build.yml

@@ -45,7 +45,8 @@ jobs:
       - name: Test
         run: |
           python3 test/run_tests.py -f --clean_dirty -E 0.001
-          ./build/test/unit_test/vw-unit-test.out
+          cd build
+          ctest --verbose --output-on-failure --label-regex VWTestList
   build_vendor_windows:
     name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.msvc.standalone
     runs-on: ${{matrix.os}}
@@ -104,7 +105,9 @@ jobs:
         # TODO: Look into test 67 failure
         run: python3 ${{ env.SOURCE_DIR }}/test/run_tests.py -f --clean_dirty -E 0.01 --skip_spanning_tree_tests --vw_bin_path ${{ env.CMAKE_BUILD_DIR }}/vowpalwabbit/${{ matrix.build_type }}/vw.exe --skip_test 67
       - name: Test unit tests
-        run: ${{ env.CMAKE_BUILD_DIR }}/test/unit_test/${{ matrix.build_type }}/vw-unit-test.out.exe
+        run: |
+          cd ${{ env.CMAKE_BUILD_DIR }}
+          ctest --verbose --output-on-failure --config ${{ matrix.build_type }} --label-regex VWTestList
   build_vendor_macos:
     runs-on: ${{matrix.os}}
     name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.AppleClang.standalone
@@ -130,8 +133,10 @@ jobs:
           -DVW_BOOST_MATH_SYS_DEP=Off
           -DVW_INSTALL=Off
       - name: Build
-        run: cmake --build build
+        run: |
+          cmake --build build
       - name: Test
         run: |
           python3 test/run_tests.py -f --clean_dirty -E 0.001
-          ./build/test/unit_test/vw-unit-test.out
+          cd build
+          ctest --verbose --output-on-failure --label-regex VWTestList

.scripts/linux/build-minimal.sh

@@ -9,5 +9,5 @@ cd $REPO_DIR
 # If parameter 1 is not supplied, it defaults to Release
 BUILD_CONFIGURATION=${1:-Release}
 
-cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DBUILD_FLATBUFFERS=On
-cmake --build build --target vw-bin vw-unit-test.out vw_c_api_unit_test spanning_tree to_flatbuff
+cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DBUILD_FLATBUFFERS=On -DVW_UNIT_TEST_WITH_VALGRIND_INTERNAL=On
+cmake --build build

.scripts/linux/build-with-coverage.sh

@@ -7,4 +7,4 @@ REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
 cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DWARNINGS=OFF -DBUILD_JAVA=Off -DBUILD_PYTHON=Off -DBUILD_TESTING=On -DBUILD_FLATBUFFERS=On
-cmake --build build --target vw-bin spanning_tree vw-unit-test.out
+cmake --build build --target vw-bin spanning_tree vw-unit-test.out vw_slim_test

.scripts/linux/test-privacy_activation.sh

@@ -7,7 +7,7 @@ REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
 cd test
-python3 run_tests.py -f --skip_spanning_tree_tests -j $(nproc) --test_spec privacy_activation.vwtest.json
+python3 run_tests.py -f --skip_spanning_tree_tests --test_spec privacy_activation.vwtest.json
 
 cd ../build
 ./test/unit_test/vw-unit-test.out --run_test=test_feature_is_activated*,test_feature_not_activated*,test_feature_could_be_activated_but_feature_not_initialized*

.scripts/linux/test-with-coverage.sh

@@ -6,6 +6,8 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
-cd test
+cd $REPO_DIR/build
+ctest --verbose --output-on-failure --label-regex VWTestList
+
+cd $REPO_DIR/test
 python3 run_tests.py -f -j $(nproc) --include_flatbuffers --clean_dirty -E 0.001
-../build/test/unit_test/vw-unit-test.out

.scripts/linux/test.sh

@@ -6,8 +6,12 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
-cd test
-python3 run_tests.py -f --epsilon 1e-3 --include_flatbuffers
+# Run unit tests
+cd $REPO_DIR/build
+ctest --verbose --output-on-failure --label-regex VWTestList
 
-cd ../build
-ctest --verbose --output-on-failure
+# Run integration tests
+cd $REPO_DIR
+python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --include_flatbuffers
+python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread --include_flatbuffers
+python3 test/run_tests.py --ignore_dirty --test_spec test/slow.vwtest.json --timeout 180

.scripts/linux/unit-tests-valgrind.sh

@@ -7,5 +7,4 @@ REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
 cd build
-valgrind --quiet --error-exitcode=100 --track-origins=yes --leak-check=full ./test/unit_test/vw-unit-test.out
-valgrind --quiet --error-exitcode=100 --track-origins=yes --leak-check=full ./bindings/c/test/vw_c_api_unit_test
+ctest --verbose --output-on-failure --label-regex VWTestList

CMakeLists.txt

@@ -330,12 +330,6 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING)
     set(cmake_test_verbosity "")
   endif()
 
-  # This target ensures all dependencies are built and also uses verbose mode allowing the test output to be seen.
-  add_custom_target(test_with_output
-    COMMAND ${CMAKE_CTEST_COMMAND} ${cmake_test_configuration} ${cmake_test_verbosity} --output-on-failure
-    DEPENDS spanning_tree vw-unit-test.out vw-bin ${vw_c_api_unit_test_target} ${vw_cs_unittest_targets}
-  )
-
   # Don't offer these make dependent targets on Windows
   if(NOT WIN32)
     # make bigtests BIG_TEST_ARGS="<args here>"

Makefile

@@ -43,7 +43,7 @@ java_build:
 
 test_build:
 	@echo "vw running test-suite..."
-	cd build; make -j$(shell cat ./build/nprocs.txt) test_with_output
+	cd build; make -j$(shell cat ./build/nprocs.txt) all; make test
 
 unit_test_build:
 	cd build/test/unit_test; make -j$(shell cat ./build/nprocs.txt) vw-unit-test.out test

cmake/VowpalWabbitUtils.cmake

@@ -1,6 +1,11 @@
 include(CMakeParseArguments)
 include(GNUInstallDirs)
 
+option(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL "Internal flag." OFF)
+if(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL)
+  find_program(VALGRIND "valgrind" REQUIRED)
+endif()
+
 # Given a lib name writes to OUTPUT what the correspinding target name will be
 function(vw_get_lib_target OUTPUT LIB_NAME)
   set(${OUTPUT} vw_${LIB_NAME} PARENT_SCOPE)
@@ -224,6 +229,11 @@ function(vw_add_test_executable)
       gmock
     )
     target_compile_definitions(${FULL_TEST_NAME} PRIVATE ${VW_TEST_COMPILE_DEFS})
-    add_test(NAME ${FULL_TEST_NAME} COMMAND ${FULL_TEST_NAME})
+    if(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL)
+      add_test(NAME ${FULL_TEST_NAME} COMMAND ${VALGRIND} $<TARGET_FILE:${FULL_TEST_NAME}>)
+    else()
+      add_test(NAME ${FULL_TEST_NAME} COMMAND ${FULL_TEST_NAME})
+    endif()
+    set_tests_properties(${FULL_TEST_NAME} PROPERTIES LABELS "VWTestList")
   endif()
 endfunction()

test/CMakeLists.txt

@@ -1,37 +1,2 @@
 add_subdirectory(unit_test)
 add_subdirectory(tools)
-
-set(FB_FLAG "") # flatbuffers are off by default
-if(BUILD_FLATBUFFERS)
-  set(FB_FLAG "--include_flatbuffers") # include flatbuffer tests in RunTests
-endif()
-
-# Don't offer target on Windows
-if(NOT WIN32)
-  add_test(
-    NAME RunTests_pass_1_onethread
-    COMMAND python3 run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread ${FB_FLAG}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  )
-
-  add_test(
-    NAME RunTests_pass_2
-    COMMAND python3 run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty ${FB_FLAG}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  )
-
-  add_test(
-    NAME slow_tests
-    COMMAND python3 run_tests.py --ignore_dirty --test_spec ${CMAKE_CURRENT_SOURCE_DIR}/slow.vwtest.json --timeout 180
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  )
-
-  if(BUILD_PRIVACY_ACTIVATION)
-    add_test(
-      NAME privacy_activation_tests
-
-      COMMAND python3 run_tests.py --ignore_dirty --test_spec ${CMAKE_CURRENT_SOURCE_DIR}/privacy_activation.vwtest.json
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    )
-  endif()
-endif()

test/unit_test/CMakeLists.txt

@@ -96,3 +96,4 @@ add_test(
   COMMAND $<TARGET_FILE:vw-unit-test.out>
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
+set_tests_properties(vw_unit_test PROPERTIES LABELS "VWTestList")