test: move last of tests to gtest, remove all boost test infra (Vowpa…

…lWabbit#4399)

* test: move last of tests to gtest

* cmake if check

* fix warnings

* merge remove_boost_unit_tests

* remove dir

* Update cs_fb.stderr

* add message to notify where tests are now located

* fix warning

* update impl of cast unsigned to signed

* gtest_discover_tests and increase timeout

* fix slim test selection and run in parallel

* bad include

* up valgrind timeout to 30s

* filter when using preset

* revert gtest_main

* don't read/write to disk in _test_helper_save_load

* exclude slow tests

* keep spin off but extend timeout

* name test w iterations

.github/workflows/asan.yml

@@ -57,15 +57,14 @@ jobs:
       - name: Build
         run: cmake --build build
       - name: Run unit tests
-        run: |
-          cd build
-          ctest --verbose --output-on-failure
+        working-directory: build
+        run: ctest --output-on-failure --no-tests=error --exclude-regex w_iterations --label-regex VWTestList
       - name: Run python test script
         if: ${{ success() || failure() }}
         run: python3 test/run_tests.py -f --clean_dirty -E 0.001 --include_flatbuffers --jobs 1
 
   test_ubsan_minimal:
-  # To avoid running out of disk space, build only VW executables and Boost unit tests on Linux
+    # To avoid running out of disk space, build only VW executables and some unit tests on Linux
     strategy:
       fail-fast: false
       matrix:
@@ -95,9 +94,9 @@ jobs:
           cmakeListsTxtPath: "${{ github.workspace }}/CMakeLists.txt"
           configurePreset: "${{ matrix.preset }}"
       - name: Build
-        run: cmake --build build --target vw_cli_bin spanning_tree vw-unit-test.out
+        run: cmake --build build -t vw_cli_bin spanning_tree vw_core_test
       - name: Run unit tests
-        run: ./build/test/unit_test/vw-unit-test.out --run_test='!*w_iterations*'
+        run: ./build/vowpalwabbit/core/vw_core_test --gtest_filter=-\*w_iterations
       - name: Run python test script
         if: ${{ success() || failure() }}
         run: python3 test/run_tests.py -f --clean_dirty -E 0.001 --include_flatbuffers

.github/workflows/build_macos.yml

@@ -9,7 +9,7 @@ on:
     branches:
       - '*'
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
@@ -30,10 +30,13 @@ jobs:
         run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DWARNINGS=Off -DVW_BUILD_VW_C_WRAPPER=Off -DBUILD_TESTING=On -DBUILD_EXPERIMENTAL_BINDING=On -DVW_BUILD_CSV=On -DVW_INSTALL=Off
       - name: Build
         run: cmake --build build --target all
-      - name: Test
+      - name: Unit tests
+        working-directory: build
         run: |
           # Run unit tests
-          ctest --test-dir build --extra-verbose --output-on-failure --label-regex VWTestList
+          ctest --output-on-failure --no-tests=error --label-regex VWTestList --parallel 2
+      - name: Test
+        run: |
           # Run integration tests
           python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty
           python3 test/run_tests.py --fuzzy_compare --exit_first_fail --epsilon 0.001 --ignore_dirty --extra_options=--onethread

.github/workflows/build_vw_slim.yml

@@ -9,7 +9,7 @@ on:
     branches:
       - '*'
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
@@ -27,4 +27,5 @@ jobs:
         run: ./.scripts/linux/build-slim.sh
       - name: Test VW Slim
         shell: bash
-        run: ./.scripts/linux/test-slim.sh
+        working-directory: build
+        run: ctest --output-on-failure --no-tests=error --tests-regex "VowpalWabbitSlim|ExploreTestSuite|CommandLineOptions" --parallel 2

.github/workflows/build_windows_cmake.yml

@@ -9,7 +9,7 @@ on:
     branches:
       - '*'
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
@@ -52,14 +52,13 @@ jobs:
       - name: Bootstrap vcpkg
         run: ${{ env.VCPKG_ROOT }}/bootstrap-vcpkg.bat
       - name: Install vcpkg dependencies
-        run: ${{ env.VCPKG_ROOT }}/vcpkg.exe --triplet x64-windows install zlib boost-test flatbuffers
+        run: ${{ env.VCPKG_ROOT }}/vcpkg.exe --triplet x64-windows install zlib flatbuffers
       - name: Generate project files
         run: |
           cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DVCPKG_MANIFEST_MODE=OFF -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DBUILD_FLATBUFFERS=On -DVW_BUILD_CSV=On -Dvw_BUILD_NET_FRAMEWORK=On
       - name: Build project
         run: |
           cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }} --verbose
       - name: Run tests
-        run: |
-          cd ${{ env.CMAKE_BUILD_DIR }}
-          ctest --verbose --output-on-failure -C ${{ matrix.build_config }} --label-regex VWTestList
+        working-directory: ${{ env.CMAKE_BUILD_DIR }}
+        run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --build-config ${{ matrix.build_config }}

.github/workflows/upload_coverage.yml

@@ -9,7 +9,7 @@ on:
     branches:
       - '*'
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 

.github/workflows/valgrind.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - '*'
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
@@ -20,7 +20,9 @@ jobs:
         with:
           submodules: recursive
       - name: Build C++ VW binary
-        run: ./.scripts/linux/build-minimal.sh Release
+        run: |
+          cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DBUILD_FLATBUFFERS=On -DVW_BUILD_CSV=On
+          cmake --build build
       - name: Upload vw binary
         uses: actions/upload-artifact@v2
         with:
@@ -40,7 +42,8 @@ jobs:
           path: build/utl/flatbuffer/to_flatbuff
           if-no-files-found: error
       - name: Run unit tests with Valgrind
-        run: ./.scripts/linux/unit-tests-valgrind.sh
+        working-directory: build
+        run: ctest --output-on-failure --exclude-regex "w_iterations" --label-regex VWTestList --timeout 10 -T memcheck --overwrite MemoryCheckCommandOptions="--leak-check=full --error-exitcode=100" --parallel 2
   run-valgrind-tests:
     name: ubuntu2004.amd64.valgrind-test.[${{ matrix.test-segment }}]
     needs: build-valgrind

.github/workflows/vendor_build.yml

@@ -9,8 +9,8 @@ on:
     branches:
       - '*'
 
-concurrency: 
-  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 jobs:
@@ -49,13 +49,12 @@ jobs:
           -DWARNING_AS_ERROR=On
           -DUSE_LATEST_STD=On
       - name: Build
-        run: |
-          cmake --build build
+        run: cmake --build build
+      - name: Unit tests
+        working-directory: build
+        run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --parallel 2
       - name: Test
-        run: |
-          python3 test/run_tests.py -f --clean_dirty -E 0.001
-          cd build
-          ctest --verbose --output-on-failure --label-regex VWTestList
+        run: python3 test/run_tests.py -f --clean_dirty -E 0.001
   build_vendor_windows:
     name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.msvc.standalone
     runs-on: ${{matrix.os}}
@@ -66,39 +65,15 @@ jobs:
     env:
       CMAKE_BUILD_DIR: ${{ github.workspace }}/vw/build
       SOURCE_DIR: ${{ github.workspace }}/vw
-      VCPKG_ROOT: ${{ github.workspace }}/vcpkg
-      VCPKG_REF: 6ca56aeb457f033d344a7106cb3f9f1abf8f4e98
     steps:
       - uses: actions/checkout@v2
         with:
           path: 'vw'
           submodules: 'recursive'
-      - uses: actions/checkout@v2
-        with:
-          path: 'vcpkg'
-          repository: 'microsoft/vcpkg'
-          ref: ${{ env.VCPKG_REF }}
-      - name: Restore vcpkg and build artifacts
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ env.VCPKG_ROOT }}/installed/
-            ${{ env.VCPKG_ROOT }}
-            !${{ env.VCPKG_ROOT }}/buildtrees
-            !${{ env.VCPKG_ROOT }}/packages
-            !${{ env.VCPKG_ROOT }}/downloads
-          key: |
-            ${{ env.VCPKG_REF }}-${{matrix.os}}-standalone-vcpkg-cache-invalidate-0
       - uses: ilammy/msvc-dev-cmd@v1
-      - name: Bootstrap vcpkg
-        run: ${{ env.VCPKG_ROOT }}/bootstrap-vcpkg.bat
-      - name: Install vcpkg dependencies
-        run: ${{ env.VCPKG_ROOT }}/vcpkg.exe --triplet x64-windows install boost-test
       - name: Configure
         run: >
           cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64"
-          -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake"
-          -DVCPKG_MANIFEST_MODE=OFF
           -DUSE_LATEST_STD=On
           -DBUILD_FLATBUFFERS=Off
           -DVW_BUILD_CSV=On
@@ -109,14 +84,12 @@ jobs:
           -DVW_BOOST_MATH_SYS_DEP=Off
           -DVW_INSTALL=Off
       - name: Build
-        run: |
-          cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_type }}
+        run: cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_type }}
       - name: Test run_tests.py
         run: python3 ${{ env.SOURCE_DIR }}/test/run_tests.py -f --clean_dirty -E 0.01 --skip_spanning_tree_tests --vw_bin_path ${{ env.CMAKE_BUILD_DIR }}/vowpalwabbit/cli/${{ matrix.build_type }}/vw.exe
       - name: Test unit tests
-        run: |
-          cd ${{ env.CMAKE_BUILD_DIR }}
-          ctest --verbose --output-on-failure --config ${{ matrix.build_type }} --label-regex VWTestList
+        working-directory: ${{ github.workspace }}/vw/build
+        run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --build-config ${{ matrix.build_type }} --parallel 2
   build_vendor_macos:
     runs-on: ${{matrix.os}}
     name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.AppleClang.standalone
@@ -129,7 +102,7 @@ jobs:
         with:
           submodules: 'recursive'
       - name: Install dependencies
-        run: brew install cmake boost ninja
+        run: brew install cmake ninja
       - name: Configure
         run: >
           cmake -S . -B build -G Ninja
@@ -143,10 +116,9 @@ jobs:
           -DVW_BOOST_MATH_SYS_DEP=Off
           -DVW_INSTALL=Off
       - name: Build
-        run: |
-          cmake --build build
+        run: cmake --build build
+      - name: Unit tests
+        working-directory: build
+        run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --parallel 2
       - name: Test
-        run: |
-          python3 test/run_tests.py -f --clean_dirty -E 0.001
-          cd build
-          ctest --verbose --output-on-failure --label-regex VWTestList
+        run: python3 test/run_tests.py -f --clean_dirty -E 0.001

.scripts/linux/test-with-coverage.sh

@@ -7,7 +7,7 @@ REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
 cd $REPO_DIR/build
-ctest --verbose --output-on-failure --label-regex VWTestList
+ctest --verbose --output-on-failure --label-regex VWTestList --parallel 2
 
 cd $REPO_DIR/test
 python3 run_tests.py -f -j $(nproc) --include_flatbuffers --clean_dirty -E 0.001

.scripts/linux/test.sh

@@ -8,7 +8,7 @@ cd $REPO_DIR
 
 # Run unit tests
 cd $REPO_DIR/build
-ctest --verbose --output-on-failure --label-regex VWTestList
+ctest --verbose --output-on-failure --label-regex VWTestList --parallel 2
 
 # Run integration tests
 cd $REPO_DIR

.scripts/restore.cmd

@@ -11,6 +11,6 @@ REM CD out of the repo dir as we need to avoid vcpkg recognizing the manifest
 cd ..
 
 REM TODO: This really should be out-of-source
-%VCPKG_INSTALLATION_ROOT%\vcpkg install flatbuffers:x64-windows boost-test:x64-windows
+%VCPKG_INSTALLATION_ROOT%\vcpkg install flatbuffers:x64-windows
 
 ENDLOCAL

.scripts/test.cmd

@@ -7,14 +7,6 @@ SETLOCAL
 
 CALL %~dp0init.cmd
 
-ECHO Running VW Unit Tests in C++
-"%vwRoot%\build\binaries\Release\vw-unit-test.out.exe"
-
-IF "%ERRORLEVEL%" NEQ "0" (
-    ENDLOCAL
-    EXIT /B %ERRORLEVEL%
-)
-
 ECHO Running VW Unit Tests in C#
 REM TODO: Add explicit logging configuration so it can be uploaded to pipeline results.
 "%vstestPath%" /Platform:x64 /inIsolation "%vwRoot%\build\binaries\Release\cs_unittest.dll" /TestCaseFilter:"TestCategory!=NotOnVSO" --logger:trx "--ResultsDirectory:%vwRoot%\vowpalwabbit\out\test\Release\x64"

CMakeLists.txt

@@ -171,29 +171,6 @@ if(DEFINED BUILD_TESTS)
   message(WARNING "Value of BUILD_TESTS option ignored. Please use the standard option BUILD_TESTING.")
 endif()
 
-if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
-  include(CTest)
-  if(BUILD_TESTING)
-    if(${CMAKE_VERSION} VERSION_LESS "3.11.0")
-      message(WARNING "BUILD_TESTING requires CMake >= 3.11.0. You can turn if off by setting BUILD_TESTING=OFF")
-    endif()
-    if(VW_GTEST_SYS_DEP)
-      find_package(GTest REQUIRED)
-    else()
-      cmake_minimum_required(VERSION 3.11)
-      include(FetchContent)
-      FetchContent_Declare(
-        googletest
-        URL https://github.com/google/googletest/archive/refs/tags/release-1.11.0.zip
-      )
-      # For Windows: Prevent overriding the parent project's compiler/linker settings
-      set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
-      set(INSTALL_GTEST OFF CACHE BOOL "" FORCE)
-      FetchContent_MakeAvailable(googletest)
-    endif()
-  endif()
-endif()
-
 if(VW_INSTALL AND NOT FMT_SYS_DEP)
   message(WARNING "Installing with a vendored version of fmt is not recommended. Use FMT_SYS_DEP to use a system dependency or specify VW_INSTALL=OFF to silence this warning.")
 endif()
@@ -272,8 +249,6 @@ if(BUILD_BENCHMARKS)
 endif()
 
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING)
-  # Boost test is only required if we are building tests
-  find_package(Boost REQUIRED COMPONENTS unit_test_framework)
   add_subdirectory(test)
 
   # Don't offer these make dependent targets on Windows