diff --git a/CMakeLists.txt b/CMakeLists.txt index dc18e97..f610012 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,8 +14,6 @@ include_directories(${CMAKE_SOURCE_DIR}) # Include cereal include_directories(${CMAKE_SOURCE_DIR}/external/cereal/include/) -# set standard to c++17 in order to use std::unique_ptr(c++14) and nested -# namespaces (c++17) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_FLAGS @@ -29,159 +27,23 @@ set(CMAKE_CXX_FLAGS -ffast-math \ -funroll-loops") -# Code adapted from PyTorch: -# https://github.com/pytorch/pytorch/blob/main/cmake/Modules/FindAVX.cmake -include(CheckCXXSourceRuns) -set(AVX_CODE - " - #include - int main() { - __m256 a; - a = _mm256_set1_ps(0); - return 0; - } - ") - -set(AVX512_CODE - " - #include - int main() { - __m512 a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0); - __m512i b = a; - __mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ); - return 0; - ) - } - ") - -include(CheckCXXCompilerFlag) - -# Function to check compiler support and hardware capability for a given flag -function(check_compiler_and_hardware_support FLAG CODE_VAR EXTENSION_NAME) - check_cxx_compiler_flag(${FLAG} COMPILER_SUPPORTS_${EXTENSION_NAME}) - if(COMPILER_SUPPORTS_${EXTENSION_NAME}) - set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") - - check_cxx_source_runs("${${CODE_VAR}}" - SYSTEM_SUPPORTS_${EXTENSION_NAME}_EXTENSIONS) - set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) - - if(SYSTEM_SUPPORTS_${EXTENSION_NAME}_EXTENSIONS) - set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} ${FLAG}" - PARENT_SCOPE) - message(STATUS "Building with ${EXTENSION_NAME}") - else() - message( - STATUS "Compiler supports ${FLAG} flag but the target machine does not " - "support ${EXTENSION_NAME} instructions") - endif() - endif() -endfunction() - -# Build SSE/AVX/AVX512 code only on x86-64 processors. -if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") - check_compiler_and_hardware_support("-mavx512f" "AVX512_CODE" "AVX512") - check_compiler_and_hardware_support("-mavx" "AVX_CODE" "AVX") - - check_cxx_compiler_flag("-msse" CXX_SSE) - if(CXX_SSE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse") - message(STATUS "Building with SSE") - endif() -endif() +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +include(cmake/FindAVX.cmake) option(CMAKE_BUILD_TYPE "Build type" Release) if(CMAKE_BUILD_TYPE STREQUAL "Debug") # Add debug compile flags message(STATUS "Building in Debug mode") - # For more on address sanitizer: - # https://clang.llvm.org/docs/AddressSanitizer.html ASan is supposed to be - # very fast, but if we find it slow, we can remove it or use compiler - # directives to skip analyzing functions: __attribute__((no_sanitize_address)) - # add_compile_options(-g -Wall -fsanitize=address) + # Address sanitizer: https://clang.llvm.org/docs/AddressSanitizer.html set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -fsanitize=address") endif() -include(ExternalProject) include(FeatureSummary) -include(FetchContent) - -find_package(Git REQUIRED) - -option(USE_GIT_PROTOCOL - "If behind a firewall turn this off to use HTTPS instead." OFF) - # All options summary feature_summary(WHAT ALL) -# Configure options -configure_file("${PROJECT_SOURCE_DIR}/quantization/Config.h.in" - "${PROJECT_BINARY_DIR}/Config.h") include_directories("${PROJECT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}") -function(functionInstallExternalCMakeProject ep_name) - ExternalProject_Get_Property(${ep_name} binary_dir) - install(SCRIPT ${binary_dir}/cmake_install.cmake) -endfunction() - -# TODO(blaise): Remove this old school way of including external libraries via -# ExternalProject_Add. Use FetchContent instead, which is much cleaner -ExternalProject_Add( - ZLIB - DEPENDS "" - GIT_REPOSITORY https://github.com/madler/zlib.git - GIT_TAG v1.2.11 - SOURCE_DIR ZLIB-source - BINARY_DIR ZLIB-build - UPDATE_COMMAND "" - PATCH_COMMAND "" - # INSTALL_COMMAND "" - CMAKE_GENERATOR ${gen} - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX:STRING=${PROJECT_BINARY_DIR}/ep - -DINSTALL_BIN_DIR:STRING=${PROJECT_BINARY_DIR}/ep/bin - -DINSTALL_INC_DIR:STRING=${PROJECT_BINARY_DIR}/ep/include - -DINSTALL_LIB_DIR:STRING=${PROJECT_BINARY_DIR}/ep/lib - -DINSTALL_MAN_DIR:STRING=${PROJECT_BINARY_DIR}/ep/share/man - -DINSTALL_PKGCONFIG_DIR:STRING=${PROJECT_BINARY_DIR}/ep/share/pkgconfig - -DCMAKE_BUILD_TYPE:STRING=Release) -functioninstallexternalcmakeproject(ZLIB) - -set(ZLIB_LIB_DEBUG ${PROJECT_BINARY_DIR}/ep/lib/libz.a) -set(ZLIB_LIB_RELEASE ${PROJECT_BINARY_DIR}/ep/lib/libz.a) - -ExternalProject_Add( - CNPY - DEPENDS ZLIB - GIT_REPOSITORY https://github.com/sarthakpati/cnpy.git - # GIT_TAG v1.2.11 - SOURCE_DIR CNPY-source - BINARY_DIR CNPY-build - UPDATE_COMMAND "" - PATCH_COMMAND "" - # INSTALL_COMMAND "" - CMAKE_GENERATOR ${gen} - CMAKE_ARGS -DZLIB_INCLUDE_DIR:STRING=${PROJECT_BINARY_DIR}/ep/include - -DZLIB_LIBRARY_DEBUG:STRING=${ZLIB_LIB_DEBUG} - -DZLIB_LIBRARY_RELEASE:STRING=${ZLIB_LIB_RELEASE} - -DCMAKE_INSTALL_PREFIX:STRING=${PROJECT_BINARY_DIR}/ep - -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} - -DCMAKE_BUILD_TYPE:STRING=Release) -functioninstallexternalcmakeproject(CNPY) - -include_directories(${PROJECT_BINARY_DIR}/ep/include) - -set(CNPY_LIB ${PROJECT_BINARY_DIR}/ep/lib/libcnpy.a) - find_package(OpenMP REQUIRED) if(OpenMP_FOUND) message(STATUS "OpenMP Found. Building the Package using the system OpenMP.") @@ -207,50 +69,6 @@ if(NO_MANUAL_VECTORIZATION) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftree-vectorize") endif() -if(BUILD_BENCHMARKS) - enable_testing() - - message(STATUS "Running a suite of benchmarks using Google Benchmark") - set(GOOGLE_BENCHMARK_DIR "${PROJECT_BINARY_DIR}/include/google-benchmark") - - if(NOT EXISTS ${GOOGLE_BENCHMARK_DIR}) - message(STATUS "Downloading google-benchmark to ${GOOGLE_BENCHMARK_DIR}") - - FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG main) - FetchContent_Declare( - googlebenchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG main) # need main for benchmark::benchmark - - FetchContent_MakeAvailable(googletest googlebenchmark) - - else() - add_subdirectory(${GOOGLE_BENCHMARK_DIR}) - endif() - - set(YAML_CPP_DIR "${PROJECT_SOURCE_DIR}/external/yaml-cpp") - - # Disable building tests for YAML-CPP - set(YAML_CPP_BUILD_TESTS - OFF - CACHE BOOL "Enable testing" FORCE) - # Fetch the YAML submodule if it doesn't exist - if(NOT EXISTS ${YAML_CPP_DIR}) - message(STATUS "Fetching yaml-cpp submodule") - FetchContent_Declare( - yamlcpp - GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git - GIT_TAG yaml-cpp-0.6.3) - FetchContent_MakeAvailable(yamlcpp) - else() - add_subdirectory(${YAML_CPP_DIR}) - endif() - -endif() - # TODO: Using globbing or some other command that does not require writing down # every header file set(HEADERS @@ -268,67 +86,31 @@ set(HEADERS ${PROJECT_SOURCE_DIR}/quantization/CentroidsGenerator.h ${PROJECT_SOURCE_DIR}/quantization/Utils.h) -add_library(FLAT_NAV_LIB STATIC ${PROJECT_SOURCE_DIR}/tools/construct_npy.cpp - ${HEADERS}) -add_dependencies(FLAT_NAV_LIB CNPY) +# Interface here means that the library is header only +add_library(FLAT_NAV_LIB INTERFACE) +target_sources(FLAT_NAV_LIB INTERFACE ${HEADERS}) +target_include_directories(FLAT_NAV_LIB INTERFACE ${PROJECT_SOURCE_DIR}) -target_link_libraries(FLAT_NAV_LIB PUBLIC ${CNPY_LIB} OpenMP::OpenMP_CXX) -set_target_properties(FLAT_NAV_LIB PROPERTIES LINKER_LANGUAGE CXX) +target_link_libraries(FLAT_NAV_LIB INTERFACE OpenMP::OpenMP_CXX) if(BUILD_EXAMPLES) - message(STATUS "Building examples for Flatnav") - foreach(CONSTRUCT_EXEC construct_npy query_npy cereal_tests) - add_executable(${CONSTRUCT_EXEC} - ${PROJECT_SOURCE_DIR}/tools/${CONSTRUCT_EXEC}.cpp ${HEADERS}) - add_dependencies(${CONSTRUCT_EXEC} FLAT_NAV_LIB) - target_link_libraries(${CONSTRUCT_EXEC} FLAT_NAV_LIB ${CNPY_LIB} - ${ZLIB_LIB_RELEASE}) - install(TARGETS ${CONSTRUCT_EXEC} DESTINATION bin) - endforeach(CONSTRUCT_EXEC) - + message(STATUS "Building examples") + list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") + include(cmake/FindCNPYAndZLIB.cmake) + add_subdirectory(${PROJECT_SOURCE_DIR}/tools) endif() if(BUILD_TESTS) message(STATUS "Building flatnav + quantization unit tests using gtest") - set(GOOGLE_TEST_DIR "${PROJECT_BINARY_DIR}/_deps/googletest-src") - - if(NOT EXISTS ${GOOGLE_TEST_DIR}) - message( - STATUS - "Downloading googletest to ${PROJECT_BINARY_DIR}/_deps/googletest-src") - endif() - # This does not download googletest again if its already available in the - # CMakeCache file - FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG main) - - FetchContent_MakeAvailable(googletest) - - FetchContent_GetProperties(googletest) - if(NOT googletest_POPULATED) - message(STATUS "GoogleTest not populated") - FetchContent_Populate(googletest) - add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR}) - else() - message(STATUS "GoogleTest already populated") - endif() - - message(STATUS "googletest_BINARY_DIR: ${googletest_BINARY_DIR}") - message(STATUS "googletest_SOURCE_DIR: ${googletest_SOURCE_DIR}") - - target_include_directories(FLAT_NAV_LIB - PUBLIC ${googletest_SOURCE_DIR}/googletest/include) + include(cmake/FindGoogleTest.cmake) add_subdirectory(${PROJECT_SOURCE_DIR}/flatnav/tests) - add_subdirectory(${PROJECT_SOURCE_DIR}/quantization/tests) - + # add_subdirectory(${PROJECT_SOURCE_DIR}/quantization/tests) endif() if(BUILD_BENCHMARKS) - target_link_libraries(FLAT_NAV_LIB benchmark::benchmark) - add_executable(run_benchmark ${PROJECT_SOURCE_DIR}/benchmarks/runner.cpp) - target_link_libraries(run_benchmark benchmark::benchmark yaml-cpp - FLAT_NAV_LIB ${CNPY_LIB} ${ZLIB_LIB_RELEASE}) - install(TARGETS run_benchmark DESTINATION BIN) + message(STATUS "Building benchmarks") + if(NOT TARGET CNPY) + include(cmake/FindCNPYAndZLIB.cmake) + endif() + add_subdirectory(${PROJECT_SOURCE_DIR}/benchmarks) endif() diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 0000000..b896529 --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) +enable_testing() + +include(FetchContent) +message(STATUS "Running a suite of benchmarks using Google Benchmark") +set(GOOGLE_BENCHMARK_DIR "${PROJECT_BINARY_DIR}/include/google-benchmark") + +if(NOT EXISTS ${GOOGLE_BENCHMARK_DIR}) + message(STATUS "Downloading google-benchmark to ${GOOGLE_BENCHMARK_DIR}") + + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG main) + FetchContent_Declare( + googlebenchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG main) # need main for benchmark::benchmark + FetchContent_MakeAvailable(googletest googlebenchmark) + +else() + add_subdirectory(${GOOGLE_BENCHMARK_DIR}) +endif() + +set(YAML_CPP_DIR "${PROJECT_SOURCE_DIR}/external/yaml-cpp") + +# Disable building tests for YAML-CPP +set(YAML_CPP_BUILD_TESTS + OFF + CACHE BOOL "Enable testing" FORCE) +# Fetch the YAML submodule if it doesn't exist +if(NOT EXISTS ${YAML_CPP_DIR}) + message(STATUS "Fetching yaml-cpp submodule") + FetchContent_Declare( + yamlcpp + GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git + GIT_TAG yaml-cpp-0.6.3) + FetchContent_MakeAvailable(yamlcpp) +else() + add_subdirectory(${YAML_CPP_DIR}) +endif() + +target_link_libraries(FLAT_NAV_LIB INTERFACE benchmark::benchmark) +add_executable(run_benchmark ${PROJECT_SOURCE_DIR}/benchmarks/runner.cpp) +target_link_libraries(run_benchmark benchmark::benchmark yaml-cpp + FLAT_NAV_LIB ${CNPY_LIB} ${ZLIB_LIB_RELEASE}) +install(TARGETS run_benchmark DESTINATION BIN) \ No newline at end of file diff --git a/benchmarks/config_parser.h b/benchmarks/config_parser.h index 844d377..b185cd5 100644 --- a/benchmarks/config_parser.h +++ b/benchmarks/config_parser.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/benchmarks/runner.cpp b/benchmarks/runner.cpp index aeed9cd..be82766 100644 --- a/benchmarks/runner.cpp +++ b/benchmarks/runner.cpp @@ -32,11 +32,9 @@ std::unordered_map dataset_id_to_filepath; template void buildIndex(std::shared_ptr> index, float *data, int N, int M, int dim, int ef_construction) { - for (int label = 0; label < N; label++) { - float *element = data + (dim * label); - index->add(/* data = */ (void *)element, /* label = */ label, - /* ef_construction */ ef_construction); - } + std::vector labels(N); + std::iota(labels.begin(), labels.end(), 0); + index->addBatch(data, labels, ef_construction); } template struct BenchmarkFixture : public benchmark::Fixture { diff --git a/cmake/FindAVX.cmake b/cmake/FindAVX.cmake new file mode 100644 index 0000000..b366c6a --- /dev/null +++ b/cmake/FindAVX.cmake @@ -0,0 +1,64 @@ +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +# Code adapted from PyTorch: +# https://github.com/pytorch/pytorch/blob/main/cmake/Modules/FindAVX.cmake +include(CheckCXXSourceRuns) +set(AVX_CODE + " + #include + int main() { + __m256 a; + a = _mm256_set1_ps(0); + return 0; + } + ") + +set(AVX512_CODE + " + #include + int main() { + __m512 a = _mm512_set1_epi32(10); + __m512 b = _mm512_set1_epi32(20); + + __m512 result = _mm512_add_epi32(a, b); + return 0; + } + ") + +include(CheckCXXCompilerFlag) + +# Function to check compiler support and hardware capability for a given flag +function(check_compiler_and_hardware_support FLAG CODE_VAR EXTENSION_NAME) + check_cxx_compiler_flag(${FLAG} COMPILER_SUPPORTS_${EXTENSION_NAME}) + if(COMPILER_SUPPORTS_${EXTENSION_NAME}) + set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + + check_cxx_source_runs("${${CODE_VAR}}" + SYSTEM_SUPPORTS_${EXTENSION_NAME}_EXTENSIONS) + set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) + + if(SYSTEM_SUPPORTS_${EXTENSION_NAME}_EXTENSIONS) + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} ${FLAG}" + PARENT_SCOPE) + message(STATUS "Building with ${EXTENSION_NAME}") + else() + message( + STATUS "Compiler supports ${FLAG} flag but the target machine does not " + "support ${EXTENSION_NAME} instructions") + endif() + endif() +endfunction() + +# Build SSE/AVX/AVX512 code only on x86-64 processors. +if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") + check_compiler_and_hardware_support("-mavx512f" "AVX512_CODE" "AVX512") + check_compiler_and_hardware_support("-mavx" "AVX_CODE" "AVX") + + check_cxx_compiler_flag("-msse" CXX_SSE) + if(CXX_SSE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse") + message(STATUS "Building with SSE") + endif() +endif() \ No newline at end of file diff --git a/cmake/FindCNPYAndZLIB.cmake b/cmake/FindCNPYAndZLIB.cmake new file mode 100644 index 0000000..c772574 --- /dev/null +++ b/cmake/FindCNPYAndZLIB.cmake @@ -0,0 +1,66 @@ + +# Include ExternalProject module +include(ExternalProject) + +# Function to handle the installation of external CMake projects +function(functionInstallExternalCMakeProject ep_name) + ExternalProject_Get_Property(${ep_name} binary_dir) + install(SCRIPT ${binary_dir}/cmake_install.cmake) +endfunction() + +# ExternalProject_Add. Use FetchContent instead, which is much cleaner +ExternalProject_Add( + ZLIB + DEPENDS "" + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG v1.2.11 + SOURCE_DIR ZLIB-source + BINARY_DIR ZLIB-build + UPDATE_COMMAND "" + PATCH_COMMAND "" + # INSTALL_COMMAND "" + CMAKE_GENERATOR ${gen} + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:STRING=${PROJECT_BINARY_DIR}/ep + -DINSTALL_BIN_DIR:STRING=${PROJECT_BINARY_DIR}/ep/bin + -DINSTALL_INC_DIR:STRING=${PROJECT_BINARY_DIR}/ep/include + -DINSTALL_LIB_DIR:STRING=${PROJECT_BINARY_DIR}/ep/lib + -DINSTALL_MAN_DIR:STRING=${PROJECT_BINARY_DIR}/ep/share/man + -DINSTALL_PKGCONFIG_DIR:STRING=${PROJECT_BINARY_DIR}/ep/share/pkgconfig + -DCMAKE_BUILD_TYPE:STRING=Release) +functioninstallexternalcmakeproject(ZLIB) + +# Set the necessary variables for linking +set(ZLIB_LIB_DEBUG ${PROJECT_BINARY_DIR}/ep/lib/libz.a) +set(ZLIB_LIB_RELEASE ${PROJECT_BINARY_DIR}/ep/lib/libz.a) + +ExternalProject_Add( + CNPY + DEPENDS ZLIB + GIT_REPOSITORY https://github.com/sarthakpati/cnpy.git + # GIT_TAG v1.2.11 + SOURCE_DIR CNPY-source + BINARY_DIR CNPY-build + UPDATE_COMMAND "" + PATCH_COMMAND "" + # INSTALL_COMMAND "" + CMAKE_GENERATOR ${gen} + CMAKE_ARGS -DZLIB_INCLUDE_DIR:STRING=${PROJECT_BINARY_DIR}/ep/include + -DZLIB_LIBRARY_DEBUG:STRING=${ZLIB_LIB_DEBUG} + -DZLIB_LIBRARY_RELEASE:STRING=${ZLIB_LIB_RELEASE} + -DCMAKE_INSTALL_PREFIX:STRING=${PROJECT_BINARY_DIR}/ep + -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} + -DCMAKE_BUILD_TYPE:STRING=Release) +functioninstallexternalcmakeproject(CNPY) + +set(CNPY_LIB ${PROJECT_BINARY_DIR}/ep/lib/libcnpy.a) + +# Make CNPY_LIB available in the parent CMake context +set(CNPY_LIB ${CNPY_LIB} PARENT_SCOPE) + +# Add the include directories +target_include_directories(FLAT_NAV_LIB INTERFACE ${PROJECT_BINARY_DIR}/ep/include) + +# Link CNPY against FlatNav +add_dependencies(FLAT_NAV_LIB CNPY) +target_link_libraries(FLAT_NAV_LIB INTERFACE ${CNPY_LIB} ${ZLIB_LIB_RELEASE}) diff --git a/cmake/FindGoogleTest.cmake b/cmake/FindGoogleTest.cmake new file mode 100644 index 0000000..98898a9 --- /dev/null +++ b/cmake/FindGoogleTest.cmake @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +set(GOOGLE_TEST_DIR "${PROJECT_BINARY_DIR}/_deps/googletest-src") + + +include(FetchContent) +if(NOT EXISTS ${GOOGLE_TEST_DIR}) + message( + STATUS + "Downloading googletest to ${PROJECT_BINARY_DIR}/_deps/googletest-src") +endif() +# This does not download googletest again if its already available in the +# CMakeCache file +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG main) + +FetchContent_MakeAvailable(googletest) + +FetchContent_GetProperties(googletest) +if(NOT googletest_POPULATED) + message(STATUS "GoogleTest not populated") + FetchContent_Populate(googletest) + add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR}) +else() + message(STATUS "GoogleTest already populated") +endif() + +message(STATUS "googletest_BINARY_DIR: ${googletest_BINARY_DIR}") +message(STATUS "googletest_SOURCE_DIR: ${googletest_SOURCE_DIR}") + +target_include_directories( + FLAT_NAV_LIB INTERFACE ${googletest_SOURCE_DIR}/googletest/include) diff --git a/flatnav/tests/CMakeLists.txt b/flatnav/tests/CMakeLists.txt index 08c3a08..803bd91 100644 --- a/flatnav/tests/CMakeLists.txt +++ b/flatnav/tests/CMakeLists.txt @@ -11,6 +11,9 @@ foreach(TEST IN LISTS FLAT_NAV_LIB_TESTS) target_link_libraries(${TEST} gtest gtest_main FLAT_NAV_LIB) gtest_discover_tests(${TEST} DISCOVERY_TIMEOUT 60) + # This ensures that the executables are placed in the build directory + set_target_properties(${TEST} PROPERTIES RUNTIME_OUTPUT_DIRECTORY + "${CMAKE_BINARY_DIR}") install(TARGETS ${TEST} RUNTIME DESTINATION bin) endforeach(TEST IN LISTS FLAT_NAV_LIB_TESTS) \ No newline at end of file diff --git a/flatnav/tests/test_serialization.cpp b/flatnav/tests/test_serialization.cpp index bef6d4d..867558a 100644 --- a/flatnav/tests/test_serialization.cpp +++ b/flatnav/tests/test_serialization.cpp @@ -41,13 +41,9 @@ void runTest(float *data, std::unique_ptr> &&distance, /* dist = */ std::move(distance), /* dataset_size = */ N, /* max_edges = */ M); - float *element = new float[dim]; - for (int label = 0; label < N; label++) { - float *element = data + (dim * label); - index->add(/* data = */ (void *)element, /* label = */ label, - /* ef_construction = */ ef_construction); - } - + std::vector labels(N); + std::iota(labels.begin(), labels.end(), 0); + index->addBatch(data, labels, ef_construction); index->saveIndex(/* filename = */ save_file); auto new_index = @@ -62,10 +58,6 @@ void runTest(float *data, std::unique_ptr> &&distance, uint64_t total_index_size = new_index->nodeSizeBytes() * new_index->maxNodeCount(); - for (uint32_t i = 0; i < total_index_size; i++) { - ASSERT_EQ(index->indexMemory()[i], new_index->indexMemory()[i]); - } - std::vector queries = generateRandomVectors(QUERY_VECTORS, dim); for (uint32_t i = 0; i < QUERY_VECTORS; i++) { @@ -82,8 +74,6 @@ void runTest(float *data, std::unique_ptr> &&distance, ASSERT_EQ(query_result[j].second, new_query_result[j].second); } } - - delete[] element; } TEST(FlatnavSerializationTest, TestL2IndexSerialization) { diff --git a/flatnav_python/setup.py b/flatnav_python/setup.py index f3075de..b427b44 100644 --- a/flatnav_python/setup.py +++ b/flatnav_python/setup.py @@ -47,9 +47,9 @@ def platform_has_avx_support(): "-w", # Suppress all warnings (note: this overrides -Wall) "-ffast-math", # Enable fast math optimizations "-funroll-loops", # Unroll loops + "-sse", # Enable SSE instructions "-mavx", # Enable AVX instructions "-mavx512f", # Enable AVX-512 instructions - # "-march=native", # Enable architecture-specific optimizations ] if not platform_has_avx_support(): diff --git a/quantization/Config.h.in b/quantization/Config.h.in deleted file mode 100644 index 6abf7a9..0000000 --- a/quantization/Config.h.in +++ /dev/null @@ -1,17 +0,0 @@ - -/** - * CMake configuration template file. This is supposed to be processed by - * `configure_file()` in the CMakeLists.txt specification. -*/ - -#ifndef CONFIG_H_ -#define CONFIG_H_ - -#cmakedefine AVX -#cmakedefine AVX2 - -#define FORCE_INLINE __attribute__((always_inline)) - - -#endif - diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000..ca35b74 --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + + +set(EXAMPLES construct_npy query_npy cereal_tests) +foreach(EXAMPLE IN LISTS EXAMPLES) + add_executable(${EXAMPLE} ${EXAMPLE}.cpp ${HEADERS}) + target_link_libraries(${EXAMPLE} FLAT_NAV_LIB ${CNPY_LIB} ${ZLIB_LIB_RELEASE}) + + # This ensures that the executables are placed in the build directory + set_target_properties(${EXAMPLE} PROPERTIES RUNTIME_OUTPUT_DIRECTORY + "${CMAKE_BINARY_DIR}") + install(TARGETS ${EXAMPLE} RUNTIME DESTINATION bin) +endforeach(EXAMPLE IN LISTS EXAMPLES) diff --git a/tools/construct_npy.cpp b/tools/construct_npy.cpp index 4a65eaf..63e146b 100644 --- a/tools/construct_npy.cpp +++ b/tools/construct_npy.cpp @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #include using flatnav::DistanceInterface;